-
Notifications
You must be signed in to change notification settings - Fork 10
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #390 from projectcaluma/feat-upload-file-to-s3storage
Use Django storage backend to handle file-upload to s3 storage
- Loading branch information
Showing
33 changed files
with
1,307 additions
and
935 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| from django.conf import settings | ||
| from django.core.files.storage import get_storage_class | ||
| from django.core.management.base import BaseCommand | ||
| from django.db import transaction | ||
| from django.db.models import Q | ||
| from tqdm import tqdm | ||
|
|
||
| from alexandria.core.models import File | ||
| from alexandria.storages.backends.s3 import SsecGlobalS3Storage | ||
|
|
||
| # This is needed to disable the warning about not verifying the SSL certificate. | ||
| # It spams the output otherwise. | ||
| if not settings.AWS_S3_VERIFY: | ||
| import urllib3 | ||
|
|
||
| urllib3.disable_warnings() | ||
|
|
||
|
|
||
| class Command(BaseCommand): | ||
| help = "Swaps plain text file content to encrypted content" | ||
|
|
||
| def add_arguments(self, parser): | ||
| parser.add_argument("--dry", dest="dry", action="store_true", default=False) | ||
|
|
||
| @transaction.atomic | ||
| def handle(self, *args, **options): | ||
| if ( | ||
| not settings.ALEXANDRIA_ENABLE_AT_REST_ENCRYPTION | ||
| or settings.ALEXANDRIA_ENCRYPTION_METHOD | ||
| == File.EncryptionStatus.NOT_ENCRYPTED.value | ||
| ): | ||
| return self.stdout.write( | ||
| self.style.WARNING( | ||
| "Encryption is not enabled. Skipping encryption of files." | ||
| ) | ||
| ) | ||
| # disable checksums to prevent errors | ||
| checksum = settings.ALEXANDRIA_ENABLE_CHECKSUM | ||
| settings.ALEXANDRIA_ENABLE_CHECKSUM = False | ||
|
|
||
| sid = transaction.savepoint() | ||
|
|
||
| # flip between default and encrypted storage to have the correct parameters in the requests | ||
| DefaultStorage = get_storage_class() | ||
| for file in tqdm( | ||
| File.objects.filter( | ||
| Q(encryption_status=File.EncryptionStatus.NOT_ENCRYPTED) | ||
| | Q(encryption_status__isnull=True) | ||
| ), | ||
| ): | ||
| # get original file content | ||
| file.content.storage = DefaultStorage() | ||
| content = file.content.open() | ||
|
|
||
| if not options["dry"]: | ||
| # overwrite with encrypted content | ||
| file.content.storage = SsecGlobalS3Storage() | ||
| file.content.save(file.content.name, content) | ||
|
|
||
| # set encryption status | ||
| file.encryption_status = settings.ALEXANDRIA_ENCRYPTION_METHOD | ||
| file.save() | ||
|
|
||
| settings.ALEXANDRIA_ENABLE_CHECKSUM = checksum | ||
| if options["dry"]: # pragma: no cover | ||
| transaction.savepoint_rollback(sid) | ||
| else: | ||
| transaction.savepoint_commit(sid) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| # Generated by Django 3.2.23 on 2023-12-13 18:00 | ||
|
|
||
| from django.db import migrations, models | ||
|
|
||
| import alexandria.core.models | ||
| import alexandria.storages.fields | ||
|
|
||
|
|
||
| def migrate_file_references(apps, schema_editor): | ||
| """Migrate the download_url based content access to storage backend access. | ||
| The simple object storage's object_name was simply the file's name. Setting | ||
| the name to the file name in the storage attribute will refer retrieve | ||
| the same object if it still exists. | ||
| Just make sure to point the object storage client to the right endpoint | ||
| and bucket before accessing the file. | ||
| """ | ||
| File = apps.get_model("alexandria_core", "File") | ||
| for file in File.objects.iterator(): | ||
| file.content.name = alexandria.core.models.upload_file_content_to(file, None) | ||
| file.save() | ||
|
|
||
|
|
||
| class Migration(migrations.Migration): | ||
| dependencies = [ | ||
| ("alexandria_core", "0012_tag_uuid_schema"), | ||
| ] | ||
|
|
||
| operations = [ | ||
| migrations.RemoveField( | ||
| model_name="file", | ||
| name="upload_status", | ||
| ), | ||
| migrations.AddField( | ||
| model_name="file", | ||
| name="content", | ||
| field=alexandria.storages.fields.DynamicStorageFileField( | ||
| default="", upload_to=alexandria.core.models.upload_file_content_to | ||
| ), | ||
| preserve_default=False, | ||
| ), | ||
| migrations.AddField( | ||
| model_name="file", | ||
| name="encryption_status", | ||
| field=models.CharField( | ||
| blank=True, | ||
| choices=[ | ||
| (None, "Encryption status not set"), | ||
| ("none", "No at-rest enryption"), | ||
| ("ssec-global", "SSE-C global key encryption (AES256)"), | ||
| ("ssec-object", "SSE-C per object encryption (AES256)"), | ||
| ], | ||
| default=None, | ||
| max_length=12, | ||
| null=True, | ||
| ), | ||
| ), | ||
| migrations.RunPython(migrate_file_references, migrations.RunPython.noop), | ||
| ] |
Oops, something went wrong.