From 219fedbfffbfc78efeea4861efa9b736635b37cd Mon Sep 17 00:00:00 2001 From: aabbaq Date: Fri, 13 May 2022 23:34:53 +1000 Subject: [PATCH 1/6] Update upload_design_doc.py --- backend/upload_design_doc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/upload_design_doc.py b/backend/upload_design_doc.py index 382e17b..a0d621d 100644 --- a/backend/upload_design_doc.py +++ b/backend/upload_design_doc.py @@ -255,7 +255,7 @@ if response.status_code != 201: print('Error uploading') - for each in cfg.get_twitter_dbs: + for each in cfg.get_twitter_dbs(): response = requests.put( req_link.format(db=each, design_doc='languages'), headers={"Content-Type": "application/json"}, @@ -264,4 +264,4 @@ if response.status_code != 201: print('Error uploading') - \ No newline at end of file + From 26ad3618d55988e50cc45894ed92fc1c1f9265e2 Mon Sep 17 00:00:00 2001 From: aabbaq Date: Fri, 13 May 2022 23:35:55 +1000 Subject: [PATCH 2/6] Update config.py --- aurin_helper/config/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aurin_helper/config/config.py b/aurin_helper/config/config.py index 8824339..c52943a 100644 --- a/aurin_helper/config/config.py +++ b/aurin_helper/config/config.py @@ -175,7 +175,7 @@ 'sep_21': 'un_rate_2021_9' } -EMPLOYMENT_PATH = '/data/Aurin/Employ/unemployment.csv' +EMPLOYMENT_PATH = '/data/Aurin/Employ/employment.csv' EMPLOYMENT_INFO = { 'mar_19': 'num_2019_3', 'jun_19': 'num_2019_6', From 50722d8910cb2b45cdafc500a53432d04ae4a0da Mon Sep 17 00:00:00 2001 From: 996Worker <69796042+Blackmesa-Canteen@users.noreply.github.com> Date: Sun, 15 May 2022 13:05:35 +1000 Subject: [PATCH 3/6] Update README.md --- README.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8e69bae..0664e91 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,18 @@ # COMP90024-Cluster-and-Cloud-Computing-Assignment-2 - Cluster & Cloud Computing, Twitter analyzer + Cluster & Cloud Computing, Twitter analyzer. + +# Description +The focus of this project is toharvest tweets from Melbourne on the MRC and undertake a variety of social media data analytics +scenarios that explore liveability of Melbourne and importantly how the Twitter data can be used +alongside/compared with/augment the data available within the AURIN platform to improve our +knowledge of the liveability of Melbourne + +This project is a Cloud-based solution that exploits a multitude of virtual machines (VMs) +across the MRC for harvesting tweets through the Twitter APIs (using both the Streaming and the +Search API interfaces). + +We have produce a solution that can be run (in principle) across any node of the MRC to +harvest and store tweets and scale up/down as required. The solution includes multiple +Twitter harvesting applications for Melbourne, These harvesting application running on +the MRC together with an associated CouchDB database containing the amalgamated collection +of Tweets. The CouchDB setup is based on a cluster setup. From 61e0ee44b3779c5bd16a702ad90ee205cb2ec510 Mon Sep 17 00:00:00 2001 From: 996Worker <69796042+Blackmesa-Canteen@users.noreply.github.com> Date: Sun, 15 May 2022 13:10:18 +1000 Subject: [PATCH 4/6] Create LICENSE --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a12bd41 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 COMP90024-2022 Group 52 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From fa9a6dd65e8ed8f9642754ede8c0176c94a254f9 Mon Sep 17 00:00:00 2001 From: 996Worker <69796042+Blackmesa-Canteen@users.noreply.github.com> Date: Sun, 15 May 2022 13:20:46 +1000 Subject: [PATCH 5/6] Update README.md --- README.md | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0664e91..6bf3b39 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # COMP90024-Cluster-and-Cloud-Computing-Assignment-2 - Cluster & Cloud Computing, Twitter analyzer. -# Description +## Introduction The focus of this project is toharvest tweets from Melbourne on the MRC and undertake a variety of social media data analytics scenarios that explore liveability of Melbourne and importantly how the Twitter data can be used alongside/compared with/augment the data available within the AURIN platform to improve our @@ -16,3 +15,36 @@ harvest and store tweets and scale up/down as required. The solution includes mu Twitter harvesting applications for Melbourne, These harvesting application running on the MRC together with an associated CouchDB database containing the amalgamated collection of Tweets. The CouchDB setup is based on a cluster setup. + +## Tech stack +- Python + Flask +- Tweepy +- CouchDB +- React.js +- Docker +- Nginx + +## License +``` +MIT License + +Copyright (c) 2022 COMP90024-2022 Group 52 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` From 079e5fbc3237a48c8b2b5eef32c9f4d969f62232 Mon Sep 17 00:00:00 2001 From: 996Worker <69796042+Blackmesa-Canteen@users.noreply.github.com> Date: Sun, 15 May 2022 13:23:09 +1000 Subject: [PATCH 6/6] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6bf3b39..be12dc2 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ of Tweets. The CouchDB setup is based on a cluster setup. - React.js - Docker - Nginx +- NLTK + TextBlob ## License ```