1 SQL_SCRIPT_DIR="tools/migrations" 2 3 DOCKER_SERVICE_NAME="repodiff-service" 4 DOCKER_CONTAINER_REGISTRY="gcr.io" 5 DOCKER_IMAGE_NAME="repodiff-image" 6 DOCKER_TAG_NAME="latest" 7 DOCKER_DOCKERFILE_DIR="." 8 DOCKER_TARGET_PORT=$(shell cat config.json | jq ".port") 9 DOCKER_CLUSTER_NAME="repodiff-default-cluster" 10 DOCKER_DEPLOYMENT_NAME="repodiff-deployment" 11 DOCKER_REPLICA_COUNT="1" 12 13 DOCKER_CANONICAL_ID=$(DOCKER_CONTAINER_REGISTRY)/$(GOOGLE_PROJECT_ID)/$(DOCKER_IMAGE_NAME):$(DOCKER_TAG_NAME) 14 15 PORT_HTTP="80" 16 GCE_ZONE="us-west1-b" 17 GCE_IMAGE_PROJECT="ubuntu-os-cloud" 18 GCE_IMAGE_FAMILY="ubuntu-1604-lts" 19 20 TMP_CREDENTIAL_FNAME=service_account_credentials.json 21 22 # https://cloud.google.com/compute/docs/machine-types 23 GCE_MACHINE_TYPE="n1-standard-64" 24 25 PROJECT_NAME="auto-diff-android-branches" 26 REMOTE_MACHINE_NAME=mithalop5 27 FIREWALL_NAME=public-http-access 28 DISK_SIZE=500GB 29 RUN_COMMAND_REMOTE=gcloud compute --project $(PROJECT_NAME) ssh --zone $(GCE_ZONE) "$(REMOTE_MACHINE_NAME)" --command 30 SCP_TO_HOST=gcloud compute --project $(PROJECT_NAME) scp --zone $(GCE_ZONE) 31 SERVICE_ACCOUNT_ID=repo-reader 32 SERVICE_ACCOUNT=$(SERVICE_ACCOUNT_ID)@$(PROJECT_NAME).iam.gserviceaccount.com 33 34 ifeq ($(ROLE),prod) 35 GCP_DB_USER=$(GCP_DB_USER_PROD) 36 GCP_DB_INSTANCE_CONNECTION_NAME=$(GCP_DB_INSTANCE_CONNECTION_NAME_PROD) 37 GCP_DB_PASSWORD=$(GCP_DB_PASSWORD_PROD) 38 GCP_DB_NAME=$(GCP_DB_NAME_PROD) 39 GCP_DB_PROXY_PORT=$(GCP_DB_PROXY_PORT_PROD) 40 else 41 GCP_DB_USER=$(GCP_DB_USER_DEV) 42 GCP_DB_INSTANCE_CONNECTION_NAME=$(GCP_DB_INSTANCE_CONNECTION_NAME_DEV) 43 GCP_DB_PASSWORD=$(GCP_DB_PASSWORD_DEV) 44 GCP_DB_NAME=$(GCP_DB_NAME_DEV) 45 GCP_DB_PROXY_PORT=$(GCP_DB_PROXY_PORT_DEV) 46 endif 47 48 49 bootstrap: 50 mkdir -p $(GOPATH)/src 51 ./tools/setup_go_path_symlink.sh 52 # include $GOPATH/bin as part of system path 53 grep -q -F 'export PATH=$$PATH:$$GOPATH/bin' ~/.bashrc || echo 'export PATH=$$PATH:$$GOPATH/bin' >> ~/.bashrc 54 source ~/.bashrc 55 cd $(GOPATH)/src/repodiff; go get github.com/GoogleCloudPlatform/cloudsql-proxy/cmd/cloud_sql_proxy; \ 56 go get github.com/golang/dep/cmd/dep; \ 57 dep ensure; \ 58 go build 59 60 run: 61 go build; 62 ROLE="dev" ./repodiff 63 64 run_prod: 65 go build; 66 ROLE="prod" ./repodiff 67 68 reformat: 69 go fmt . 70 71 db_shell: 72 mysql -u $(GCP_DB_USER) -h 127.0.0.1 -P $(GCP_DB_PROXY_PORT) -p$(GCP_DB_PASSWORD) $(GCP_DB_NAME) ${EXTRA} 73 74 db_proxy: 75 $(GOPATH)/bin/cloud_sql_proxy -instances=$(GCP_DB_INSTANCE_CONNECTION_NAME_DEV)=tcp:$(GCP_DB_PROXY_PORT_DEV) & 76 $(GOPATH)/bin/cloud_sql_proxy -instances=$(GCP_DB_INSTANCE_CONNECTION_NAME_PROD)=tcp:$(GCP_DB_PROXY_PORT_PROD) & 77 78 db_proxy_ignore_err: 79 make db_proxy; true 80 81 start_sql: 82 make db_proxy_ignore_err & 83 make db_shell < $(SQL_SCRIPT_DIR)/required_meta.sql 84 85 db_upgrade: 86 make start_sql 87 python tools/upgrade_db.py upgrade $(SQL_SCRIPT_DIR) 88 89 90 db_downgrade: 91 make start_sql 92 python tools/upgrade_db.py downgrade $(SQL_SCRIPT_DIR) 93 94 example: 95 make db_shell EXTRA="-e 'DESCRIBE project_differential;'" 96 97 test: 98 rm -rf build/ 99 ROLE="dev" go test ./... -v | grep -v PASS | grep -v RUN 100 101 sql_script: 102 python tools/create_sql_script.py 103 104 ssh: 105 gcloud compute --project $(PROJECT_NAME) ssh --zone $(GCE_ZONE) $(REMOTE_MACHINE_NAME) 106 107 deploy: 108 gcloud config set project $(PROJECT_NAME) 109 @echo "Starting docker image build" 110 make build_container_image 111 @echo "Creating machine if it doesn't already exist" 112 gcloud compute instances create $(REMOTE_MACHINE_NAME) \ 113 --machine-type $(GCE_MACHINE_TYPE) \ 114 --boot-disk-size $(DISK_SIZE) \ 115 --scopes https://www.googleapis.com/auth/source.read_only,https://www.googleapis.com/auth/compute \ 116 --zone $(GCE_ZONE) \ 117 --local-ssd interface=nvme \ 118 --metadata-from-file startup-script=remote_scripts/gce_startup.sh \ 119 --metadata AUTHOR=$(USER),SERVICE_ACCOUNT=$(SERVICE_ACCOUNT),GOOGLE_PROJECT_ID=$(GOOGLE_PROJECT_ID) \ 120 --image-project $(GCE_IMAGE_PROJECT) \ 121 --image-family $(GCE_IMAGE_FAMILY) \ 122 --min-cpu-platform skylake \ 123 --service-account $(SERVICE_ACCOUNT) \ 124 2>/dev/null || true 125 @echo "Hackily waiting a bit for instance to start up" 126 # TODO(slobdell) need to add a mechanism to block until startup script has completed 127 @sleep 60 128 ./tools/clear_service_account_keys.py $(SERVICE_ACCOUNT) 2>/dev/null || true 129 gcloud iam service-accounts keys create $(TMP_CREDENTIAL_FNAME) --iam-account $(SERVICE_ACCOUNT) 130 $(RUN_COMMAND_REMOTE) 'mkdir -p /tmp/scripts' 131 $(SCP_TO_HOST) remote_scripts/* "$(REMOTE_MACHINE_NAME)":/tmp/scripts/ 132 $(SCP_TO_HOST) $(TMP_CREDENTIAL_FNAME) "$(REMOTE_MACHINE_NAME)":/tmp/ 133 rm $(TMP_CREDENTIAL_FNAME) 134 @echo "Stopping all existing docker images" 135 $(RUN_COMMAND_REMOTE) 'docker stop $$(docker ps -a -q)' 2>/dev/null || true 136 docker image save $(DOCKER_CANONICAL_ID) -o transferrable_docker_image.tar \ 137 && $(SCP_TO_HOST) transferrable_docker_image.tar "$(REMOTE_MACHINE_NAME)":~/transferred_docker_image.tar \ 138 && $(RUN_COMMAND_REMOTE) 'docker load -i transferred_docker_image.tar' \ 139 && $(RUN_COMMAND_REMOTE) 'docker run -d --rm -p $(DOCKER_TARGET_PORT):$(DOCKER_TARGET_PORT) $(DOCKER_CANONICAL_ID)' \ 140 && gcloud compute firewall-rules create $(FIREWALL_NAME) --allow tcp:$(DOCKER_TARGET_PORT) 2>/dev/null || true \ 141 && gcloud compute firewall-rules update $(FIREWALL_NAME) --allow tcp:$(DOCKER_TARGET_PORT) --source-tags="$(REMOTE_MACHINE_NAME)" --source-ranges=0.0.0.0/0 --description="Allow requests over HTTP" 142 @make output_instance_url --no-print-directory 143 @rm transferrable_docker_image.tar 144 145 output_instance_url: 146 @echo "Monitor progress at http://"$(shell (gcloud compute instances list | grep $(REMOTE_MACHINE_NAME) | awk -F ' ' '{print $$5}')):$(DOCKER_TARGET_PORT)/health 147 148 destroy: 149 gcloud compute instances delete $(REMOTE_MACHINE_NAME) --zone $(GCE_ZONE) --quiet 150 151 ############## DOCKER DEPLOYMENT 152 build_container_image: 153 mkdir -p ./build 154 # move contents of entire directory into build 155 find . -type f -not -path 'build' -exec cp --parents '{}' 'build/' \; 156 # copy python scripts repo uses, TODO re-structure codebase so the dependencies align with file structure 157 cp ../../*.{txt,py} build/ 158 # Application credentials must be downloaded from https://pantheon.corp.google.com; set this environment variable to the path of the downloaded file 159 cp $(GOOGLE_APPLICATION_CREDENTIALS) build/ 160 # copy local environment variables into Dockerfile 161 cat Dockerfile | envsubst > build/Dockerfile 162 # copy permissions from local gitcookies into Dockerfile (container will sync repo as you) 163 cp static/docker_git_config ./build/.gitconfig 164 cp ~/.gitcookies ./build/.gitcookies 165 docker build -t $(DOCKER_CANONICAL_ID) ./build 166 rm -rf ./build 167 168 docker_shell: 169 docker run -it --rm $(DOCKER_CANONICAL_ID) bash 170 171 upload_container_image: 172 gcloud config set project $(GOOGLE_PROJECT_ID) 173 gcloud docker -- push $(DOCKER_CANONICAL_ID) 174 175 run_container_local: 176 docker run --rm -p $(DOCKER_TARGET_PORT):$(DOCKER_TARGET_PORT) $(DOCKER_CANONICAL_ID) 177 178 create_container_cluster: 179 gcloud container clusters create $(DOCKER_CLUSTER_NAME) \ 180 --num-nodes=3 \ 181 --machine-type $(GCE_MACHINE_TYPE) \ 182 --zone $(GCE_ZONE) 183 184 create_container_cluster_verify: 185 gcloud compute instances list 186 187 expose_to_internets: 188 kubectl expose deployment $(DOCKER_DEPLOYMENT_NAME) --type=LoadBalancer --port $(PORT_HTTP) --target-port $(DOCKER_TARGET_PORT) 189 190 expose_to_internets_verify: 191 kubectl get service 192 193 scale: 194 kubectl scale deployment $(DOCKER_DEPLOYMENT_NAME) --replicas=$(DOCKER_REPLICA_COUNT) 195 196 cleanup: 197 kubectl delete service 198 ############## END DOCKER DEPLOYMENT 199
