1 # 2 # Copyright (C) 2016 The Android Open Source Project 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 # 16 """Class to fetch artifacts from internal build server. 17 """ 18 19 import googleapiclient 20 import httplib2 21 import io 22 import json 23 import logging 24 import re 25 import time 26 from googleapiclient.discovery import build 27 from oauth2client import client as oauth2_client 28 from oauth2client.service_account import ServiceAccountCredentials 29 30 logger = logging.getLogger('artifact_fetcher') 31 32 33 class DriverError(Exception): 34 """Base Android GCE driver exception.""" 35 36 37 class AndroidBuildClient(object): 38 """Client that manages Android Build. 39 40 Attributes: 41 service: object, initialized and authorized service object for the 42 androidbuildinternal API. 43 API_NAME: string, name of internal API accessed by the client. 44 API_VERSION: string, version of the internal API accessed by the client. 45 SCOPE: string, URL for which to request access via oauth2. 46 DEFAULT_RESOURCE_ID: string, default artifact name to request. 47 DEFAULT_ATTEMPT_ID: string, default attempt to request for the artifact. 48 DEFAULT_CHUNK_SIZE: int, number of bytes to download at a time. 49 RETRY_COUNT: int, max number of retries. 50 RETRY_DELAY_IN_SECS: int, time delays between retries in seconds. 51 """ 52 53 API_NAME = "androidbuildinternal" 54 API_VERSION = "v2beta1" 55 SCOPE = "https://www.googleapis.com/auth/androidbuild.internal" 56 57 # other variables. 58 BUILDS_KEY = "builds" 59 BUILD_ID_KEY = "buildId" 60 DEFAULT_ATTEMPT_ID = "latest" 61 DEFAULT_BUILD_ATTEMPT_STATUS = "complete" 62 DEFAULT_BUILD_TYPE = "submitted" 63 DEFAULT_CHUNK_SIZE = 20 * 1024 * 1024 64 DEFAULT_RESOURCE_ID = "0" 65 66 # Defaults for retry. 67 RETRY_COUNT = 5 68 RETRY_DELAY_IN_SECS = 3 69 70 def __init__(self, oauth2_service_json): 71 """Initialize. 72 73 Args: 74 oauth2_service_json: Path to service account json file. 75 """ 76 authToken = ServiceAccountCredentials.from_json_keyfile_name( 77 oauth2_service_json, [self.SCOPE]) 78 http_auth = authToken.authorize(httplib2.Http()) 79 for _ in xrange(self.RETRY_COUNT): 80 try: 81 self.service = build( 82 serviceName=self.API_NAME, 83 version=self.API_VERSION, 84 http=http_auth) 85 break 86 except oauth2_client.AccessTokenRefreshError as e: 87 # The following HTTP code typically indicates transient errors: 88 # 500 (Internal Server Error) 89 # 502 (Bad Gateway) 90 # 503 (Service Unavailable) 91 logging.exception(e) 92 logging.info("Retrying to connect to %s", self.API_NAME) 93 time.sleep(self.RETRY_DELAY_IN_SECS) 94 95 def DownloadArtifactToFile(self, 96 branch, 97 build_target, 98 build_id, 99 resource_id, 100 dest_filepath, 101 attempt_id=None): 102 """Get artifact from android build server. 103 104 Args: 105 branch: Branch from which the code was built, e.g. "master" 106 build_target: Target name, e.g. "gce_x86-userdebug" 107 build_id: Build id, a string, e.g. "2263051", "P2804227" 108 resource_id: Name of resource to be downloaded, a string. 109 attempt_id: string, attempt id, will default to DEFAULT_ATTEMPT_ID. 110 dest_filepath: string, set a file path to store to a file. 111 112 Returns: 113 Contents of the requested resource as a string if dest_filepath is None; 114 None otherwise. 115 """ 116 return self.GetArtifact(branch, build_target, build_id, resource_id, 117 attempt_id=attempt_id, dest_filepath=dest_filepath) 118 119 def GetArtifact(self, 120 branch, 121 build_target, 122 build_id, 123 resource_id, 124 attempt_id=None, 125 dest_filepath=None): 126 """Get artifact from android build server. 127 128 Args: 129 branch: Branch from which the code was built, e.g. "master" 130 build_target: Target name, e.g. "gce_x86-userdebug" 131 build_id: Build id, a string, e.g. "2263051", "P2804227" 132 resource_id: Name of resource to be downloaded, a string. 133 attempt_id: string, attempt id, will default to DEFAULT_ATTEMPT_ID. 134 dest_filepath: string, set a file path to store to a file. 135 136 Returns: 137 Contents of the requested resource as a string if dest_filepath is None; 138 None otherwise. 139 """ 140 attempt_id = attempt_id or self.DEFAULT_ATTEMPT_ID 141 api = self.service.buildartifact().get_media( 142 buildId=build_id, 143 target=build_target, 144 attemptId=attempt_id, 145 resourceId=resource_id) 146 logger.info("Downloading artifact: target: %s, build_id: %s, " 147 "resource_id: %s", build_target, build_id, resource_id) 148 fh = None 149 try: 150 if dest_filepath: 151 fh = io.FileIO(dest_filepath, mode='wb') 152 else: 153 fh = io.BytesIO() 154 155 downloader = googleapiclient.http.MediaIoBaseDownload( 156 fh, api, chunksize=self.DEFAULT_CHUNK_SIZE) 157 done = False 158 while not done: 159 _, done = downloader.next_chunk() 160 logger.info("Downloaded artifact %s" % resource_id) 161 162 if not dest_filepath: 163 return fh.getvalue() 164 except OSError as e: 165 logger.error("Downloading artifact failed: %s", str(e)) 166 raise DriverError(str(e)) 167 finally: 168 if fh: 169 fh.close() 170 171 def GetManifest(self, branch, build_target, build_id, attempt_id=None): 172 """Get Android build manifest XML file. 173 174 Args: 175 branch: Branch from which the code was built, e.g. "master" 176 build_target: Target name, e.g. "gce_x86-userdebug" 177 build_id: Build id, a string, e.g. "2263051", "P2804227" 178 attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID. 179 180 181 Returns: 182 Contents of the requested XML file as a string. 183 """ 184 resource_id = "manifest_%s.xml" % build_id 185 return self.GetArtifact(branch, build_target, build_id, resource_id, 186 attempt_id) 187 188 def GetRepoDictionary(self, 189 branch, 190 build_target, 191 build_id, 192 attempt_id=None): 193 """Get dictionary of repositories and git revision IDs 194 195 Args: 196 branch: Branch from which the code was built, e.g. "master" 197 build_target: Target name, e.g. "gce_x86-userdebug" 198 build_id: Build id, a string, e.g. "2263051", "P2804227" 199 attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID. 200 201 202 Returns: 203 Dictionary of project names (string) to commit ID (string) 204 """ 205 resource_id = "BUILD_INFO" 206 build_info = self.GetArtifact(branch, build_target, build_id, 207 resource_id, attempt_id) 208 try: 209 return json.loads(build_info)["repo-dict"] 210 except (ValueError, KeyError): 211 logger.warn("Could not find repo dictionary.") 212 return {} 213 214 def GetCoverage(self, 215 branch, 216 build_target, 217 build_id, 218 product, 219 attempt_id=None): 220 """Get Android build coverage zip file. 221 222 Args: 223 branch: Branch from which the code was built, e.g. "master" 224 build_target: Target name, e.g. "gce_x86-userdebug" 225 build_id: Build id, a string, e.g. "2263051", "P2804227" 226 product: Name of product for build target, e.g. "bullhead", "angler" 227 attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID. 228 229 230 Returns: 231 Contents of the requested zip file as a string. 232 """ 233 resource_id = ("%s-coverage-%s.zip" % (product, build_id)) 234 return self.GetArtifact(branch, build_target, build_id, resource_id, 235 attempt_id) 236 237 def ListBuildIds(self, 238 branch, 239 build_target, 240 limit=1, 241 build_type=DEFAULT_BUILD_TYPE, 242 build_attempt_status=DEFAULT_BUILD_ATTEMPT_STATUS): 243 """Get a list of most recent build IDs. 244 245 Args: 246 branch: Branch from which the code was built, e.g. "master" 247 build_target: Target name, e.g. "gce_x86-userdebug" 248 limit: (optional) an int, max number of build IDs to fetch, 249 default of 1 250 build_type: (optional) a string, the build type to filter, default 251 of "submitted" 252 build_attempt_status: (optional) a string, the build attempt status 253 to filter, default of "completed" 254 255 Returns: 256 A list of build ID strings in reverse time order. 257 """ 258 builds = self.service.build().list( 259 branch=branch, 260 target=build_target, 261 maxResults=limit, 262 buildType=build_type, 263 buildAttemptStatus=build_attempt_status).execute() 264 return [str(build.get(self.BUILD_ID_KEY)) 265 for build in builds.get(self.BUILDS_KEY)] 266
