Fs eth prod

Dockerfile

@@ -1,33 +1,27 @@
 FROM python:3.8-slim
 
-RUN set -ex \
-    && apt-get -y update && apt-get -y upgrade \
-    && apt install python3-pip -y \
-    && apt install curl -y \
+# Install dependencies
+RUN apt-get update && apt-get install -y libsasl2-dev curl gcc libldap2-dev \
     && DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker} \
     && mkdir -p $DOCKER_CONFIG/cli-plugins \
     && curl -SL https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose \
     && chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose
 
-
-ADD . /datahub
+# Set the working directory and copy the application files
 WORKDIR /datahub
+COPY . /datahub
 
-# RUN poetry init
-RUN python3 -m pip install --upgrade  pip
-RUN pip install -r requirements.txt
-
-# RUN python manage.py makemigrations \
-#     && python manage.py migrate \
-#     && python manage.py loaddata db_scripts/userrole_fixture.yaml \
-#     && python manage.py loaddata db_scripts/initial_data.yaml
+# Upgrade pip and install required Python packages
+RUN python -m pip install --upgrade pip \
+    && pip install python-ldap==3.3.1 \
+    && pip install --upgrade pyopenssl \
+    && pip install -r requirements.txt
 
-ENV  PYTHONUNBUFFERED 1
-# ENV VIRTUAL_ENV /env
-
-# ENV PATH /env/bin:$PATH
+# Set environment variables
+ENV PYTHONUNBUFFERED 1
 
+# Expose port 8000 for the Django app
 EXPOSE 8000
 
-
+# Command to run the Django development server
 CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"]

datahub/migrations/0041_resource_resourcefile.py

@@ -0,0 +1,95 @@
+# Generated by Django 4.1.5 on 2023-12-07 08:17
+
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("datahub", "0040_alter_datasetv2_name_alter_policy_description_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Resource",
+            fields=[
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                (
+                    "id",
+                    models.UUIDField(
+                        default=uuid.uuid4,
+                        editable=False,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+                ("title", models.CharField(max_length=100)),
+                ("description", models.TextField(max_length=250)),
+                ("category", models.JSONField(default=dict)),
+                (
+                    "user_map",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="datahub.userorganizationmap",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="ResourceFile",
+            fields=[
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                (
+                    "id",
+                    models.UUIDField(
+                        default=uuid.uuid4,
+                        editable=False,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+                (
+                    "file",
+                    models.FileField(
+                        blank=True, null=True, upload_to="users/resources/"
+                    ),
+                ),
+                ("file_size", models.PositiveIntegerField(blank=True, null=True)),
+                (
+                    "type",
+                    models.CharField(
+                        choices=[
+                            ("youtube", "youtube"),
+                            ("pdf", "pdf"),
+                            ("file", "file"),
+                        ],
+                        max_length=20,
+                        null=True,
+                    ),
+                ),
+                ("url", models.CharField(max_length=200, null=True)),
+                (
+                    "transcription",
+                    models.CharField(blank=True, max_length=2500, null=True),
+                ),
+                (
+                    "resource",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="resources",
+                        to="datahub.resource",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]

datahub/migrations/0042_merge_20231207_0944.py

@@ -0,0 +1,16 @@
+# Generated by Django 4.1.5 on 2023-12-07 09:44
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        (
+            "datahub",
+            "0041_langchainpgcollection_langchainpgembedding_resource_and_more",
+        ),
+        ("datahub", "0041_resource_resourcefile"),
+    ]
+
+    operations = []

datahub/migrations/0043_delete_langchainpgcollection_and_more.py

@@ -0,0 +1,28 @@
+# Generated by Django 4.1.5 on 2023-12-07 09:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("datahub", "0042_merge_20231207_0944"),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name="LangchainPgCollection",
+        ),
+        migrations.DeleteModel(
+            name="LangchainPgEmbedding",
+        ),
+        migrations.AlterField(
+            model_name="resourcefile",
+            name="type",
+            field=models.CharField(
+                choices=[("youtube", "youtube"), ("pdf", "pdf"), ("file", "file")],
+                max_length=20,
+                null=True,
+            ),
+        ),
+    ]

datahub/migrations/0044_alter_resource_description_alter_resource_title.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-12-19 18:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("datahub", "0043_delete_langchainpgcollection_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="resource",
+            name="description",
+            field=models.TextField(max_length=500),
+        ),
+        migrations.AlterField(
+            model_name="resource",
+            name="title",
+            field=models.CharField(max_length=200),
+        ),
+    ]

datahub/models.py

@@ -296,7 +296,9 @@ def __str__(self) -> str:
 
 RESOURCE_URL_TYPE = (
     ("youtube", "youtube"),
-    ("pdf", "pdf")
+    ("pdf", "pdf"),
+    ("file", "file")
+
 )
 
 class ResourceFile(TimeStampMixin):
@@ -315,34 +317,31 @@ class ResourceFile(TimeStampMixin):
 
     def __str__(self) -> str:
         return self.file.name
-from pgvector.django import VectorField
+# from pgvector.django import VectorField
 
 # class ResourceVector(TimeStampMixin):
 #     resource_file = models.ForeignKey(ResourceFile, on_delete=models.CASCADE, related_name="resource_file")
 
-class LangchainPgCollection(models.Model):
-    name = models.UUIDField()
-    cmetadata = models.JSONField()
-    uuid = models.UUIDField(primary_key=True)
-
-    class Meta:
-        db_table = 'langchain_pg_collection'
-
+# class LangchainPgCollection(models.Model):
+#     name = models.UUIDField()
+#     cmetadata = models.JSONField()
+#     uuid = models.UUIDField(primary_key=True)
 
-class LangchainPgEmbedding(models.Model):
-    # resource_file = models.ForeignKey(ResourceFile, on_delete=models.CASCADE)
-    collection_id = models.UUIDField()
-    embedding = VectorField(1563)  # Assuming 'vector' is a custom PostgreSQL data type
-    document = models.TextField()
-    cmetadata = models.JSONField()
-    custom_id = models.CharField(max_length=255)
-    uuid = models.UUIDField(primary_key=True)
+#     class Meta:
+#         db_table = 'langchain_pg_collection'
 
-    class Meta:
-        db_table = 'langchain_pg_embedding'
-
-    def __str__(self):
-        return f"LangchainPgEmbedding(uuid={self.uuid}, document={self.document})"
 
+# class LangchainPgEmbedding(models.Model):
+#     # resource_file = models.ForeignKey(ResourceFile, on_delete=models.CASCADE)
+#     collection_id = models.UUIDField()
+#     embedding = VectorField(1563)  # Assuming 'vector' is a custom PostgreSQL data type
+#     document = models.TextField()
+#     cmetadata = models.JSONField()
+#     custom_id = models.CharField(max_length=255)
+#     uuid = models.UUIDField(primary_key=True)
 
+#     class Meta:
+#         db_table = 'langchain_pg_embedding'
 
+#     def __str__(self):
+#         return f"LangchainPgEmbedding(uuid={self.uuid}, document={self.document})"

datahub/serializers.py

@@ -8,6 +8,7 @@
 from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.core.validators import URLValidator
+from django.db.models import Count, Q
 from django.utils.translation import gettext as _
 from rest_framework import serializers, status
 from django.db.models import Count