-
Notifications
You must be signed in to change notification settings - Fork 6
/
google_drive_concourse_resource_common.py
executable file
·221 lines (194 loc) · 8.5 KB
/
google_drive_concourse_resource_common.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
#!/usr/local/bin/python
from __future__ import print_function
import os,sys
from oauth2client.service_account import ServiceAccountCredentials
from httplib2 import Http
from apiclient.discovery import build
from apiclient import errors
from apiclient import http
from datetime import datetime
def getServiceInstance(sEmail, pID, cID, pKey,verbose=False):
"""Creates an authenticated service instance to use for subsequent requests to google drive
Args:
sEmail: client_email to use for service account credentials to authenticate with google drive
pID: private key id to use for service account credentials to authenticate with google drive
cID: client_id to use for service account credentials to authenticate with google drive
pKey: private key to use for service account credentials to authenticate with google drive
Returns:
An authenticated service instance of google drive v2 API
"""
scopes = ['https://www.googleapis.com/auth/drive','https://www.googleapis.com/auth/drive.file']
key_dict = {'type':'service_account','client_email': sEmail, 'private_key_id':pID , 'client_id': cID, 'private_key': pKey}
if verbose:
print (str(key_dict),file=sys.stderr)
##Adding a debug print for testing
#print(dir(ServiceAccountCredentials))
credentials = ServiceAccountCredentials.from_json_keyfile_dict(key_dict, scopes=scopes)
# Impersonate a user
#delegated_credentails = credentials.create_delegated(user + '@pivotal.io')
# Apply credential headers to all requests made by an an httlib2.Http instance
http_auth = credentials.authorize(Http())
# Build a service object for the drive API with the authenticated http instance
gdriveservice = build('drive', 'v2', http=http_auth)
oldq='0B_rb6msCq2WfSVk2QVl6UUk5cFk' # skahler's private folder with no access
return gdriveservice
def listFilesinFolder(service, folderID, fileName, verbose=False):
"""Searches a google drive folder for a file
Args:
service: google drive service instance to use
folderID: the folder to lok for the file in
fileName: name of the file to search for
verbose: print debugging information
Returns:
File ID of the file was found and none if it was not
"""
drive_service=service
if verbose:
print('Query = ' + folderID + ' in parents and name = ' + fileName + '',file=sys.stderr)
results = drive_service.files().list(q="'" + folderID + "' in parents and title = '" + fileName + "'",
corpus='DEFAULT',
spaces='drive',
maxResults=10).execute()
if verbose:
print(results,file=sys.stderr)
items = results.get('items', [])
if not items:
# print('No files found.', file=sys.stderr)
return None
else:
#print('Files:')
latest = ''
for item in items:
if verbose:
print('Name: {0} ID: ({1}) MimeType: {2} ModTime: {3}'.format(item['title'], item['id'], item['mimeType'], item['modifiedDate']),file=sys.stderr)
mTime = item['modifiedDate']
if latest is None:
latest = mTime
elif latest < mTime:
latest = mTime
fileFound = item
return fileFound
def create_folder(service, folderName, parentID = None):
"""Creates a folder to use.
Args:
service: google drive service instance to use
parentID: Parent folder's ID in which to create the new folder
folderName: name to use for the new folder
Returns:
Folder Name and ID of the newly created folder
"""
drive_service=service
body = {
'title' : folderName,
'mimeType' : 'application/vnd.google-apps.folder'
}
if parentID:
body['parents'] = [{'id': parentID}]
file = drive_service.files().insert(body=body).execute()
print ('Folder Name: {0} ID: {1}' .format(file.get('title'), file.get('id')),file=sys.stderr)
def putFile(service, folderID, filePath, verbose=False):
"""Creates a File on a google drive folder.
Args:
service: google drive service instance to use
folderID: Parent Folder's ID in which to create the new file
filePath: Path to the file that needs to be put on google drive
verbose: print debugging information
Returns:
File name and ID of the newly created File or error if error occured
To Do:
Check if File and folder exist
Possible check if File is not 0 length before putting it up on google drive
"""
drive_service=service
f=os.path.basename(filePath)
mime_type=''
# Getting Env info to use for metadata tagging
atc = os.getenv('ATC_EXTERNAL_URL','DEFAULT_ATC')
pipe = os.getenv('BUILD_PIPELINE_NAME','DEFAULT_PIPELINE')
job = os.getenv('BUILD_JOB_NAME','DEFAULT_JOB')
build = os.getenv('BUILD_NAME','DEFAULT_BUILD_NUM')
# Creteing a link URL to put in metadata
link = atc + '/pipelines/' + pipe + '/jobs/' + job + '/builds/' + build
media_body = http.MediaFileUpload(filePath,mimetype=mime_type,resumable=True)
body = {
'title': f,
'description': 'Uploaded by concourse task: \n\n' + link + '',
'mimeType': mime_type
}
# Set the parent Folder
if folderID:
body['parents'] = [{'id': folderID}]
#Temporarily adding perms here
#body['permissions'] = perms
if verbose:
#print('Body: ' .format(dir(body)))
print('Body: {0}' .format(body),file=sys.stderr)
try:
file = drive_service.files().insert(
body=body,
media_body=media_body).execute()
#Printing for debug only
#print ('File ID {0} ' .format(file.get('id')))
return file
except errors.HttpError, error:
#print 'An error occured: %s' % error
return error
# Commenting out as we don't need this
# def getPermissions(service, file_id):
# """Retrieve a list of permissions.
# Args:
# service: Drive API service instance.
# file_id: ID of the file to retrieve permissions for.
# Returns:
# List of permissions.
# """
# try:
# permissions = service.permissions().list(fileId=file_id).execute()
# return permissions.get('items', [])
# except errors.HttpError, error:
# print ('An error occurred: {0}' .format(error))
# return error
def getFile(service, folderID, fileID, fileName, destPath, verbose=False):
"""Retrieves a File from a google drive Folder. Currently we only
support binary files so you can not get docs,spreadhseets, etc.
Args:
service: google drive service instance to use
folderID: Parent Folder's ID from which to get the file
fileID: Id of the file that needs to be retrieved from google drive
fileName: Name to use for the local file
destPath: destination path to use for the local file
verbose: print debugging information
Returns:
File that was requested or error if error occured
To Do:
Check if File and folder exist
Possible check if File is not 0 length before putting it up on google drive
"""
drive_service=service
os.chdir(destPath)
local_fd=open(fileName,'w')
if verbose:
print('Received FileID = ' + fileID , file=sys.stderr)
print('Destination received: {0}'.format(destPath),file=sys.stderr)
request = drive_service.files().get_media(fileId=fileID)
alternateLinkFilePath= os.path.join(destPath, 'filelink.txt')
fileUrlRequest = drive_service.files().get(fileId=fileID).execute()
fileDownloadUrl = fileUrlRequest['webContentLink']
if verbose:
print('Headers: {0}' .format(request.headers),file=sys.stderr)
print('URI: {0}' .format(request.uri),file=sys.stderr)
media_request = http.MediaIoBaseDownload(local_fd, request)
while True:
try:
download_progress, done = media_request.next_chunk()
except errors.HttpError, error:
#print 'An error occurred: %s' % error
return error
if download_progress:
print('Download Progress: %d%%' % int(download_progress.progress() * 100), file=sys.stderr)
if done:
#print 'Download Complete'
with open(alternateLinkFilePath, 'w') as fh:
fh.write(fileDownloadUrl)
print('Download Artifacts: {0}'.format(fileDownloadUrl), file=sys.stderr)
return