diff --git a/Makefile b/Makefile index 293a60c..147e254 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,14 @@ run-dev: GIT_REVISION=$(shell git rev-parse --short HEAD) \ pipenv run ./manage.py runserver 8889 +run-dev-standalone: + docker-compose -f docker-compose.dev.yml up -d && \ + GIT_TAG=$(shell git describe --tags) \ + GIT_BRANCH=$(shell git rev-parse --abbrev-ref HEAD) \ + GIT_REVISION=$(shell git rev-parse --short HEAD) \ + pipenv run ./manage.py runserver 8889 + + build-docker-image: docker build \ -t c2dhunilu/journal-of-digital-history-backend:${BUILD_TAG} \ diff --git a/Pipfile b/Pipfile index 0300675..6891820 100644 --- a/Pipfile +++ b/Pipfile @@ -9,11 +9,11 @@ verify_ssl = true psycopg2-binary = "==2.8.6" djangorestframework = "==3.12.2" django-crispy-forms = "==1.10.0" -asgiref = "==3.3.1" +asgiref = "==3.3.2" pytz = "==2020.4" sqlparse = "==0.4.1" -drf-recaptcha = "==2.0.4" -Django = "==3.1.3" +drf-recaptcha = "==2.2.1" +Django = "==3.2.14" django-ipware = "==3.0.2" shortuuid = "==1.0.1" jsonschema = "==3.2.0" @@ -44,7 +44,12 @@ jupyter-core = "==4.7.1" traitlets = "==5.0.5" urllib3 = "==1.26.6" ipython_genutils = "==0.2.0" -citeproc-py = "*" +citeproc-py = "==0.6.0" +lxml = "==4.6.3" +pillow = "==8.1.2" +pycountry = "==22.3.5" +qrcode = "==7.3" +django-import-export = "==2.8.0" [requires] python_version = "3.8" diff --git a/Pipfile.lock b/Pipfile.lock index a7739da..1e44ed6 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "e501e534f8bbbd4d3bfb364526a7f7fd5a470450caaae996f5f45af514df02be" + "sha256": "953b12302d6f5f2abf707f46dfb05e58a4249ee8c54e4c7756beb581e4bb65c1" }, "pipfile-spec": 6, "requires": { @@ -26,11 +26,11 @@ }, "asgiref": { "hashes": [ - "sha256:5ee950735509d04eb673bd7f7120f8fa1c9e2df495394992c73234d526907e17", - "sha256:7162a3cb30ab0609f1a4c95938fd73e8604f63bdba516a7f7d64b83ff09478f0" + "sha256:34103fa20270d8843a66e5df18547d2e8139534d23e3beffe96647c65ddffd4d", + "sha256:c62b616b226d6c2e927b0225f8101f9e2cca08112cff98839ca6726c129ff9e0" ], "index": "pypi", - "version": "==3.3.1" + "version": "==3.3.2" }, "attrs": { "hashes": [ @@ -111,13 +111,29 @@ "index": "pypi", "version": "==0.1.6" }, + "defusedxml": { + "hashes": [ + "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", + "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.7.1" + }, + "diff-match-patch": { + "hashes": [ + "sha256:8bf9d9c4e059d917b5c6312bac0c137971a32815ddbda9c682b949f2986b4d34", + "sha256:da6f5a01aa586df23dfc89f3827e1cafbb5420be9d87769eeb079ddfd9477a18" + ], + "markers": "python_version >= '2.7'", + "version": "==20200713" + }, "django": { "hashes": [ - "sha256:14a4b7cd77297fba516fc0d92444cc2e2e388aa9de32d7a68d4a83d58f5a4927", - "sha256:14b87775ffedab2ef6299b73343d1b4b41e5d4e2aa58c6581f114dbec01e3f8f" + "sha256:677182ba8b5b285a4e072f3ac17ceee6aff1b5ce77fd173cc5b6a2d3dc022fcf", + "sha256:a8681e098fa60f7c33a4b628d6fcd3fe983a0939ff1301ecacac21d0b38bad56" ], "index": "pypi", - "version": "==3.1.3" + "version": "==3.2.14" }, "django-crispy-forms": { "hashes": [ @@ -135,6 +151,14 @@ "index": "pypi", "version": "==2.4.0" }, + "django-import-export": { + "hashes": [ + "sha256:31d7dcfba22251e3ef93accb7da844f58c4d78585db9dd7dae37bb76f939da2c", + "sha256:33c37b2921ef84e2cd9aa0eb76d04a7c2b538c9d04cb1ed97ac32600876cab30" + ], + "index": "pypi", + "version": "==2.8.0" + }, "django-ipware": { "hashes": [ "sha256:c7df8e1410a8e5d6b1fbae58728402ea59950f043c3582e033e866f0f0cf5e94" @@ -152,11 +176,19 @@ }, "drf-recaptcha": { "hashes": [ - "sha256:758d5e026516d5821799de7cdf369b13a1a13d4d80f20a2d91f35ff1f372555d", - "sha256:ded63a89a91efe41064aac9069959abb879921bf37434fb405422adb46169153" + "sha256:20e55e9c4a3cc1e6d125cc9f726df2ee8eab60fb03c3aafde38a23a835e737f3", + "sha256:9f8453214f09f88cd53f857cd3a6b3534f94603df734042f3acc4a8007e26a10" ], "index": "pypi", - "version": "==2.0.4" + "version": "==2.2.1" + }, + "et-xmlfile": { + "hashes": [ + "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c", + "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada" + ], + "markers": "python_version >= '3.6'", + "version": "==1.1.0" }, "gunicorn": { "hashes": [ @@ -257,6 +289,7 @@ "sha256:f8380c03e45cf09f8557bdaa41e1fa7c81f3ae22828e1db470ab2a6c96d8bc23", "sha256:f90ba11136bfdd25cae3951af8da2e95121c9b9b93727b1b896e3fa105b2f586" ], + "index": "pypi", "version": "==4.6.3" }, "marko": { @@ -267,6 +300,12 @@ "index": "pypi", "version": "==1.0.3" }, + "markuppy": { + "hashes": [ + "sha256:1adee2c0a542af378fe84548ff6f6b0168f3cb7f426b46961038a2bcfaad0d5f" + ], + "version": "==1.14" + }, "nbformat": { "hashes": [ "sha256:aa9450c16d29286dc69b92ea4913c1bffe86488f90184445996ccc03a2f60382", @@ -275,6 +314,59 @@ "index": "pypi", "version": "==5.0.8" }, + "odfpy": { + "hashes": [ + "sha256:db766a6e59c5103212f3cc92ec8dd50a0f3a02790233ed0b52148b70d3c438ec", + "sha256:fc3b8d1bc098eba4a0fda865a76d9d1e577c4ceec771426bcb169a82c5e9dfe0" + ], + "version": "==1.4.1" + }, + "openpyxl": { + "hashes": [ + "sha256:0ab6d25d01799f97a9464630abacbb34aafecdcaa0ef3cba6d6b3499867d0355", + "sha256:e47805627aebcf860edb4edf7987b1309c1b3632f3750538ed962bbcc3bd7449" + ], + "version": "==3.0.10" + }, + "pillow": { + "hashes": [ + "sha256:15306d71a1e96d7e271fd2a0737038b5a92ca2978d2e38b6ced7966583e3d5af", + "sha256:1940fc4d361f9cc7e558d6f56ff38d7351b53052fd7911f4b60cd7bc091ea3b1", + "sha256:1f93f2fe211f1ef75e6f589327f4d4f8545d5c8e826231b042b483d8383e8a7c", + "sha256:30d33a1a6400132e6f521640dd3f64578ac9bfb79a619416d7e8802b4ce1dd55", + "sha256:328240f7dddf77783e72d5ed79899a6b48bc6681f8d1f6001f55933cb4905060", + "sha256:46c2bcf8e1e75d154e78417b3e3c64e96def738c2a25435e74909e127a8cba5e", + "sha256:5762ebb4436f46b566fc6351d67a9b5386b5e5de4e58fdaa18a1c83e0e20f1a8", + "sha256:5a2d957eb4aba9d48170b8fe6538ec1fbc2119ffe6373782c03d8acad3323f2e", + "sha256:5cf03b9534aca63b192856aa601c68d0764810857786ea5da652581f3a44c2b0", + "sha256:5daba2b40782c1c5157a788ec4454067c6616f5a0c1b70e26ac326a880c2d328", + "sha256:63cd413ac52ee3f67057223d363f4f82ce966e64906aea046daf46695e3c8238", + "sha256:6efac40344d8f668b6c4533ae02a48d52fd852ef0654cc6f19f6ac146399c733", + "sha256:71b01ee69e7df527439d7752a2ce8fb89e19a32df484a308eca3e81f673d3a03", + "sha256:71f31ee4df3d5e0b366dd362007740106d3210fb6a56ec4b581a5324ba254f06", + "sha256:72027ebf682abc9bafd93b43edc44279f641e8996fb2945104471419113cfc71", + "sha256:74cd9aa648ed6dd25e572453eb09b08817a1e3d9f8d1bd4d8403d99e42ea790b", + "sha256:81b3716cc9744ffdf76b39afb6247eae754186838cedad0b0ac63b2571253fe6", + "sha256:8565355a29655b28fdc2c666fd9a3890fe5edc6639d128814fafecfae2d70910", + "sha256:87f42c976f91ca2fc21a3293e25bd3cd895918597db1b95b93cbd949f7d019ce", + "sha256:89e4c757a91b8c55d97c91fa09c69b3677c227b942fa749e9a66eef602f59c28", + "sha256:8c4e32218c764bc27fe49b7328195579581aa419920edcc321c4cb877c65258d", + "sha256:903293320efe2466c1ab3509a33d6b866dc850cfd0c5d9cc92632014cec185fb", + "sha256:90882c6f084ef68b71bba190209a734bf90abb82ab5e8f64444c71d5974008c6", + "sha256:98afcac3205d31ab6a10c5006b0cf040d0026a68ec051edd3517b776c1d78b09", + "sha256:a01da2c266d9868c4f91a9c6faf47a251f23b9a862dce81d2ff583135206f5be", + "sha256:aeab4cd016e11e7aa5cfc49dcff8e51561fa64818a0be86efa82c7038e9369d0", + "sha256:b07c660e014852d98a00a91adfbe25033898a9d90a8f39beb2437d22a203fc44", + "sha256:bead24c0ae3f1f6afcb915a057943ccf65fc755d11a1410a909c1fefb6c06ad1", + "sha256:d1d6bca39bb6dd94fba23cdb3eeaea5e30c7717c5343004d900e2a63b132c341", + "sha256:e2cd8ac157c1e5ae88b6dd790648ee5d2777e76f1e5c7d184eaddb2938594f34", + "sha256:e5739ae63636a52b706a0facec77b2b58e485637e1638202556156e424a02dc2", + "sha256:f36c3ff63d6fc509ce599a2f5b0d0732189eed653420e7294c039d342c6e204a", + "sha256:f91b50ad88048d795c0ad004abbe1390aa1882073b1dca10bfd55d0b8cf18ec5" + ], + "index": "pypi", + "version": "==8.1.2" + }, "prompt-toolkit": { "hashes": [ "sha256:25c95d2ac813909f813c93fde734b6e44406d1477a9faef7c915ff37d39c0a8c", @@ -324,6 +416,13 @@ "index": "pypi", "version": "==2.8.6" }, + "pycountry": { + "hashes": [ + "sha256:b2163a246c585894d808f18783e19137cb70a0c18fb36748dc01fc6f109c1646" + ], + "index": "pypi", + "version": "==22.3.5" + }, "pyrsistent": { "hashes": [ "sha256:2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e" @@ -339,6 +438,58 @@ "index": "pypi", "version": "==2020.4" }, + "pyyaml": { + "hashes": [ + "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf", + "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293", + "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b", + "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57", + "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b", + "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4", + "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07", + "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba", + "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9", + "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287", + "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513", + "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0", + "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782", + "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0", + "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92", + "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f", + "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2", + "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc", + "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1", + "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c", + "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86", + "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4", + "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c", + "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34", + "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b", + "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d", + "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c", + "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb", + "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7", + "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737", + "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3", + "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d", + "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358", + "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53", + "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78", + "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803", + "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a", + "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f", + "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174", + "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5" + ], + "version": "==6.0" + }, + "qrcode": { + "hashes": [ + "sha256:d72861b65e26b611609f0547f0febe58aed8ae229d6bf4e675834f40742915b3" + ], + "index": "pypi", + "version": "==7.3" + }, "redis": { "hashes": [ "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2", @@ -355,6 +506,14 @@ "index": "pypi", "version": "==2.25.1" }, + "setuptools": { + "hashes": [ + "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54", + "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75" + ], + "markers": "python_version >= '3.7'", + "version": "==65.6.3" + }, "shortuuid": { "hashes": [ "sha256:3c11d2007b915c43bee3e10625f068d8a349e04f0d81f08f5fa08507427ebf1f", @@ -379,6 +538,21 @@ "index": "pypi", "version": "==0.4.1" }, + "tablib": { + "extras": [ + "html", + "ods", + "xls", + "xlsx", + "yaml" + ], + "hashes": [ + "sha256:870d7e688f738531a14937a055e8bba404fbc388e77d4d500b2c904075d1019c", + "sha256:a57f2770b8c225febec1cb1e65012a69cf30dd28be810e0ff98d024768c7d0f1" + ], + "markers": "python_version >= '3.7'", + "version": "==3.2.1" + }, "traitlets": { "hashes": [ "sha256:178f4ce988f69189f7e523337a3e11d91c786ded9360174a3d9ca83e79bc5396", @@ -410,6 +584,20 @@ ], "index": "pypi", "version": "==0.2.5" + }, + "xlrd": { + "hashes": [ + "sha256:6a33ee89877bd9abc1158129f6e94be74e2679636b8a205b43b85206c3f0bbdd", + "sha256:f72f148f54442c6b056bf931dbc34f986fd0c3b0b6b5a58d013c9aef274d0c88" + ], + "version": "==2.0.1" + }, + "xlwt": { + "hashes": [ + "sha256:a082260524678ba48a297d922cc385f58278b8aa68741596a87de01a9c628b2e", + "sha256:c59912717a9b28f1a3c2a98fd60741014b06b043936dcecbc113eaaada156c88" + ], + "version": "==1.3.0" } }, "develop": {} diff --git a/README.md b/README.md index c22b397..06f9512 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,16 @@ pipenv install -r requirements.txt make run-dev ``` +There is a docker-compose.dev.yml file you can use to starts a postgres database and the redis server, with port exposed. +Credentials are given in the .env file. + +In this case use: + +``` +make run-dev-standalone +``` + + ### SEO configuration in development THe django app `jdhseo` needs a JDHSEO_PROXY_HOST env variable in the full form **without the trailing slash** like `https://local-proxy-domain-name`. @@ -20,7 +30,6 @@ The domain name provided should correctly handle the `/proxy-githubusercontent` path. By default this value is set to `https://journalofdigitalhistory.org` - ### Celery tasks in development The django app `jdhtasks` is a separate app that should contain all celery tasks. However, as we use `autodiscover_tasks`, tasks can be added in each app. diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..526cfeb --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,16 @@ +version: '3.7' +services: + redis: + image: redis:alpine + restart: always + entrypoint: redis-server --appendonly yes + ports: + - "${REDIS_PORT:-6379}:6379" + postgresdb: + image: postgres:12-alpine + ports: + - "${DATABASE_PORT:-54320}:5432" + environment: + POSTGRES_USER: ${DATABASE_USER} + POSTGRES_DB: ${DATABASE_NAME} + POSTGRES_PASSWORD: ${DATABASE_PASSWORD} \ No newline at end of file diff --git a/jdh/settings.py b/jdh/settings.py index cea34c7..db69bda 100644 --- a/jdh/settings.py +++ b/jdh/settings.py @@ -243,3 +243,6 @@ 'JDHSEO_PROXY_HOST', 'https://journalofdigitalhistory.org/') JDHSEO_PROXY_PATH_GITHUB = get_env_variable( 'JDHSEO_PROXY_PATH_GITHUB', '/proxy-githubusercontent') + +# jdhtasks +JDHTASKS_WEBHOOK_SECRET=get_env_variable('JDHTASKS_WEBHOOK_SECRET', 'toomanysecrets') \ No newline at end of file diff --git a/jdh/urls.py b/jdh/urls.py index 5056ce0..6e01315 100644 --- a/jdh/urls.py +++ b/jdh/urls.py @@ -22,4 +22,5 @@ path('', include('jdhapi.urls')), # add seo urls and views path('', include('jdhseo.urls')), + path('tasks/', include('jdhtasks.urls')), ] diff --git a/jdhtasks/urls.py b/jdhtasks/urls.py new file mode 100644 index 0000000..3f65635 --- /dev/null +++ b/jdhtasks/urls.py @@ -0,0 +1,9 @@ +from django.urls import re_path +from . import views +from rest_framework.urlpatterns import format_suffix_patterns + +urlpatterns = [ + re_path(r'^webhook/(?P\w+)/(?P\w+)/$', views.webhook), +] + +urlpatterns = format_suffix_patterns(urlpatterns) \ No newline at end of file diff --git a/jdhtasks/views.py b/jdhtasks/views.py new file mode 100644 index 0000000..084ceb6 --- /dev/null +++ b/jdhtasks/views.py @@ -0,0 +1,40 @@ +from hmac import HMAC, compare_digest +from hashlib import sha256 +from rest_framework.decorators import api_view, permission_classes +from rest_framework.permissions import AllowAny +from rest_framework.response import Response +from django.conf import settings +from rest_framework import status + + +def verify_signature(req): + received_sign = req.headers.get('X-Hub-Signature-256').split('sha256=')[-1].strip() + secret = str(settings.JDHTASKS_WEBHOOK_SECRET).encode() + expected_sign = HMAC(key=secret, msg=req.body, digestmod=sha256).hexdigest() + return compare_digest(received_sign, expected_sign) + + +@api_view([ + 'POST']) +@permission_classes([AllowAny]) +def webhook(request, username, repo): + """ Webook echo default """ + # Extract signature header + # See https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads + # test with curl: + signature = request.headers.get("X-Hub-Signature-256") + if not signature or not signature.startswith("sha256="): + return Response({ + 'error': 'X-Hub-Signature required, or not valid.' + }, status=status.HTTP_400_BAD_REQUEST) + + if not verify_signature(request): + return Response({ + 'error': 'X-Hub-Signature is definitely not valid.' + }, status=status.HTTP_400_BAD_REQUEST) + + # The signature is fine, let's parse the data + data = request.get_json() + + # We are only interested in push events from the a certain repo + return Response({'message': 'Hello!', 'username': username, 'repo': repo, 'data': data}) \ No newline at end of file