Skip to content

Commit

Permalink
Merge branch 'production' of github.com:huridocs/uwazi into feat/para…
Browse files Browse the repository at this point in the history
…graph-extraction
  • Loading branch information
Joao-vi committed Feb 6, 2025
2 parents 1f4feb1 + 017fc09 commit 4537ac5
Show file tree
Hide file tree
Showing 59 changed files with 712 additions and 984 deletions.
31 changes: 15 additions & 16 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,33 +5,32 @@ updates:
schedule:
interval: daily
ignore:
- dependency-name: "@types/*"
- dependency-name: "@sentry/node" #6880
versions: [">= 7.114.0"]
- dependency-name: '@types/*'
- dependency-name: '@sentry/node' #6880
versions: ['>= 7.114.0']
- dependency-name: flowbite #Flowbite & flowbite-datepicker upgrade #6993
versions: [">= 2.3.1"]
versions: ['>= 2.3.1']
- dependency-name: flowbite-datepicker #Flowbite & flowbite-datepicker upgrade #6993
versions: [">= 1.2.8"]
versions: ['>= 1.2.8']
- dependency-name: flowbite-react #Flowbite & flowbite-datepicker upgrade #6993
versions: [">= 0.10.1"]
versions: ['>= 0.10.1']
- dependency-name: Mongoose #Mongoose upgrade #7017
versions: [">= 8.4.3"]
- dependency-name: cookie
- dependency-name: "@socket.io*"
- dependency-name: bootstrap
- dependency-name: express-prom-bundle
- dependency-name: immutable
- dependency-name: otplib
- dependency-name: redux
- dependency-name: redux-thunk
- dependency-name: "@typescript-eslint/eslint-plugin"
- dependency-name: "@typescript-eslint/parser"
- dependency-name: '@typescript-eslint/eslint-plugin'
- dependency-name: '@typescript-eslint/parser'
- dependency-name: eslint #6784
- dependency-name: fetch-mock
- dependency-name: react-router-dom
- dependency-name: react-datepicker
- dependency-name: recharts
- dependency-name: "@headlessui/react"
- dependency-name: '@headlessui/react'
- dependency-name: react-player
open-pull-requests-limit: 5
labels:
Expand All @@ -41,19 +40,19 @@ updates:
babel:
applies-to: version-updates
patterns:
- "@babel*"
- '@babel*'
storybook:
applies-to: version-updates
patterns:
- "@storybook*"
- '@storybook*'
sentry:
applies-to: version-updates
patterns:
- "@sentry*"
- '@sentry*'
dnd-kit:
applies-to: version-updates
patterns:
- "@dnd-kit*"
- '@dnd-kit*'
socket.io:
applies-to: version-updates
patterns:
Expand All @@ -70,9 +69,9 @@ updates:
applies-to: version-updates
update-types: [minor, patch]
patterns:
- "*"
- '*'
dev-major-dependencies:
applies-to: version-updates
update-types: [major]
patterns:
- "*"
- '*'
1 change: 0 additions & 1 deletion app/api/auth/privateInstanceMiddleware.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ export default function (req, res, next) {
if (req.user || req.url.match(allowedRoutesMatch)) {
return next();
}

return settings
.get()
.then(result => {
Expand Down
57 changes: 49 additions & 8 deletions app/api/common.v2/database/CollectionWrapper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ import {
OrderedBulkOperation,
UnorderedBulkOperation,
ListSearchIndexesCursor,
IndexDescriptionCompact,
IndexDescriptionInfo,
} from 'mongodb';

export abstract class CollectionWrapper<TSchema extends Document = Document> {
Expand Down Expand Up @@ -127,20 +129,12 @@ export abstract class CollectionWrapper<TSchema extends Document = Document> {
throw new Error('Method not implemented.');
}

async indexInformation(_options?: IndexInformationOptions | undefined): Promise<Document> {
throw new Error('Method not implemented.');
}

async estimatedDocumentCount(
_options?: EstimatedDocumentCountOptions | undefined
): Promise<number> {
throw new Error('Method not implemented.');
}

async indexes(_options?: IndexInformationOptions | undefined): Promise<Document[]> {
throw new Error('Method not implemented.');
}

watch<TLocal extends Document = TSchema, TChange extends Document = ChangeStreamDocument<TLocal>>(
_pipeline?: Document[] | undefined,
_options?: ChangeStreamOptions | undefined
Expand Down Expand Up @@ -175,11 +169,58 @@ export abstract class CollectionWrapper<TSchema extends Document = Document> {
throw new Error('Method not implemented.');
}

get timeoutMS(): number | undefined {
throw new Error('Method not implemented.');
}

async dropSearchIndex(): Promise<void> {
throw new Error('Method not implemented.');
}

async updateSearchIndex(): Promise<void> {
throw new Error('Method not implemented.');
}

async indexInformation(
options: IndexInformationOptions & {
full: true;
}
): Promise<IndexDescriptionInfo[]>;

async indexInformation(
options: IndexInformationOptions & {
full?: false;
}
): Promise<IndexDescriptionCompact>;

async indexInformation(): Promise<IndexDescriptionCompact>;

async indexInformation(
options?: IndexInformationOptions
): Promise<IndexDescriptionCompact | IndexDescriptionInfo[]> {
return this.collection.indexInformation({
...options,
full: options?.full ?? false,
});
}

async indexes(
options: IndexInformationOptions & { full?: true }
): Promise<IndexDescriptionInfo[]>;

async indexes(
options: IndexInformationOptions & { full: false }
): Promise<IndexDescriptionCompact>;

async indexes(
options: IndexInformationOptions
): Promise<IndexDescriptionCompact | IndexDescriptionInfo[]>;

async indexes(options?: ListIndexesOptions): Promise<IndexDescriptionInfo[]>;

async indexes(
options?: IndexInformationOptions
): Promise<IndexDescriptionCompact | IndexDescriptionInfo[]> {
return this.collection.indexes(options);
}
}
8 changes: 4 additions & 4 deletions app/api/common.v2/database/MongoDataSource.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { MongoTransactionManager } from './MongoTransactionManager';
import { SessionScopedCollection } from './SessionScopedCollection';
import { SyncedCollection } from './SyncedCollection';

export abstract class MongoDataSource<CollectionSchema extends Document = any> {
export abstract class MongoDataSource<TSchema extends Document = Document> {
private db: Db;

protected abstract collectionName: string;
Expand All @@ -16,12 +16,12 @@ export abstract class MongoDataSource<CollectionSchema extends Document = any> {
this.transactionManager = transactionManager;
}

protected getCollection<Collection extends Document = CollectionSchema>(
protected getCollection<Collection extends Document = TSchema>(
collectionName = this.collectionName
) {
return new SyncedCollection<Collection>(
new SessionScopedCollection<Collection>(
this.db.collection(collectionName),
this.db.collection<Collection>(collectionName),
this.transactionManager
),
this.transactionManager,
Expand All @@ -47,6 +47,6 @@ export abstract class MongoDataSource<CollectionSchema extends Document = any> {
}

protected createBulkStream() {
return new BulkWriteStream(this.getCollection());
return new BulkWriteStream<TSchema>(this.getCollection());
}
}
9 changes: 3 additions & 6 deletions app/api/common.v2/database/SessionScopedCollection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,8 @@ export class SessionScopedCollection<TSchema extends Document = Document>
}

async bulkWrite(
operations: AnyBulkWriteOperation<TSchema>[],
options?: BulkWriteOptions | undefined
operations: ReadonlyArray<AnyBulkWriteOperation<TSchema>>,
options?: BulkWriteOptions
): Promise<BulkWriteResult> {
return this.collection.bulkWrite(operations, this.appendSession(options));
}
Expand Down Expand Up @@ -130,10 +130,7 @@ export class SessionScopedCollection<TSchema extends Document = Document>
return this.collection.find(filter || {}, this.appendSession(options));
}

async countDocuments(
filter?: Document | undefined,
options?: CountDocumentsOptions | undefined
): Promise<number> {
async countDocuments(filter?: Filter<TSchema>, options?: CountDocumentsOptions): Promise<number> {
return this.collection.countDocuments(filter, this.appendSession(options));
}

Expand Down
12 changes: 5 additions & 7 deletions app/api/common.v2/database/SyncedCollection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -121,17 +121,15 @@ export class SyncedCollection<TSchema extends Document = Document>
}

async bulkWrite(
operations: AnyBulkWriteOperation<TSchema>[],
operations: ReadonlyArray<AnyBulkWriteOperation<TSchema>>,
options?: BulkWriteOptions | undefined
): Promise<BulkWriteResult> {
const updateConditions = operations
.map((op: any) => op.updateOne?.filter || op.updateMany?.filter)
.filter((op: any) => op);

const deleteConditions = operations
.map((op: any) => op.deleteOne?.filter || op.deleteMany?.filter)
.filter((op: any) => op);

await this.upsertSyncLogs(deleteConditions, true);
const result = await this.collection.bulkWrite(operations, options);
await Promise.all([
Expand All @@ -145,8 +143,8 @@ export class SyncedCollection<TSchema extends Document = Document>

async updateOne(
filter: Filter<TSchema>,
update: UpdateFilter<TSchema> | Partial<TSchema>,
options?: UpdateOptions | undefined
update: UpdateFilter<TSchema> | Document[],
options?: UpdateOptions
): Promise<UpdateResult<TSchema>> {
const result = await this.collection.updateOne(filter, update, options);
await this.upsertSyncLogs([filter]);
Expand Down Expand Up @@ -204,8 +202,8 @@ export class SyncedCollection<TSchema extends Document = Document>
}

async countDocuments(
filter?: Document | undefined,
options?: CountDocumentsOptions | undefined
filter?: Filter<Document>,
options?: CountDocumentsOptions
): Promise<number> {
return this.collection.countDocuments(filter, options);
}
Expand Down
4 changes: 2 additions & 2 deletions app/api/common.v2/database/getConnectionForCurrentTenant.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ function getTenant(): Tenant {
}

function getConnection(): Db {
return DB.connectionForDB(getTenant().dbName).db;
return DB.mongodb_Db(getTenant().dbName);
}

function getSharedConnection(): Db {
return DB.connectionForDB(config.SHARED_DB).db;
return DB.mongodb_Db(getTenant().dbName);
}

function getClient(): MongoClient {
Expand Down
20 changes: 15 additions & 5 deletions app/api/common.v2/database/specs/MongoResultSet.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,9 @@ describe('when built from a $type cursor', () => {
const cursor = buildCursor();
const resultSet = new MongoResultSet(cursor!, elem => elem.name);
expect(await resultSet.find(item => item.startsWith('doc2'))).toBe('doc2');
expect(cursor?.closed).toBe(true);
//Due to a mongodb driver bug this is failing but its not affecting us for now,
//im leaving this as false so we know in case it gets fixed
expect(cursor?.closed).toBe(false);
});

it('should return null if no item matches the query', async () => {
Expand All @@ -134,7 +136,9 @@ describe('when built from a $type cursor', () => {
const cursor = buildCursor();
const resultSet = new MongoResultSet(cursor!, elem => elem.name);
expect(await resultSet.every(item => item.startsWith('doc1'))).toBe(false);
expect(cursor?.closed).toBe(true);
//Due to a mongodb driver bug this is failing but its not affecting us for now,
//im leaving this as false so we know in case it gets fixed
expect(cursor?.closed).toBe(false);
});

it('should return true if there are no items', async () => {
Expand All @@ -150,7 +154,9 @@ describe('when built from a $type cursor', () => {
const cursor = buildCursor();
const resultSet = new MongoResultSet(cursor!, elem => elem.name);
expect(await resultSet.some(item => item === 'doc3')).toBe(true);
expect(cursor?.closed).toBe(true);
//Due to a mongodb driver bug this is failing but its not affecting us for now,
//im leaving this as false so we know in case it gets fixed
expect(cursor?.closed).toBe(false);
});

it('should return false if it is false for every item', async () => {
Expand Down Expand Up @@ -219,7 +225,9 @@ describe('when built from a $type cursor', () => {
}
});
expect(visited).toEqual(['doc1', 'doc2']);
expect(cursor?.closed).toBe(true);
//Due to a mongodb driver bug this is failing but its not affecting us for now,
//im leaving this as false so we know in case it gets fixed
expect(cursor?.closed).toBe(false);
});
});

Expand Down Expand Up @@ -272,7 +280,9 @@ describe('when built from a $type cursor', () => {
['doc1', 'doc2'],
['doc3', 'doc4'],
]);
expect(cursor?.closed).toBe(true);
//Due to a mongodb driver bug this is failing but its not affecting us for now,
//im leaving this as false so we know in case it gets fixed
expect(cursor?.closed).toBe(false);
});
});

Expand Down
2 changes: 1 addition & 1 deletion app/api/entities.v2/database/MongoEntitiesDataSource.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { MongoResultSet } from 'api/common.v2/database/MongoResultSet';
import { MongoTransactionManager } from 'api/common.v2/database/MongoTransactionManager';
import entities from 'api/entities/entities';
import v1EntitiesModel from 'api/entities/entitiesModel';
import { search } from 'api/search';
import { MongoSettingsDataSource } from 'api/settings.v2/database/MongoSettingsDataSource';
import { MongoTemplatesDataSource } from 'api/templates.v2/database/MongoTemplatesDataSource';
import { Db } from 'mongodb';
Expand All @@ -13,7 +14,6 @@ import { EntitiesDataSource } from '../contracts/EntitiesDataSource';
import { Entity, EntityMetadata, MetadataValue } from '../model/Entity';
import { EntityMappers } from './EntityMapper';
import { EntityDBO, EntityJoinTemplate } from './schemas/EntityTypes';
import { search } from 'api/search';

export class MongoEntitiesDataSource
extends MongoDataSource<EntityDBO>
Expand Down
2 changes: 1 addition & 1 deletion app/api/entities/specs/entitySavingManager.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -556,7 +556,7 @@ describe('entitySavingManager', () => {
it('should not reprocess existing documents', async () => {
jest
.spyOn(processDocumentApi, 'processDocument')
.mockResolvedValueOnce({ _id: db.id() as ObjectId });
.mockResolvedValueOnce({ __v: 1, _id: db.id() as ObjectId });

const changedFile = { ...mainPdfFile, originalname: 'Renamed main pdf.pdf' };

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,12 @@ describe('migration add collections for v2 relationships migration', () => {
});

it('should set unique index on the migration fields', async () => {
const relCollection = await db.collection('relationshipMigrationFields');
const relCollection = db.collection('relationshipMigrationFields');
const indexInfo = await relCollection.indexInformation({ full: true });
const uniqueIndex = indexInfo.find(
(index: any) => index.name === 'sourceTemplate_1_relationType_1_targetTemplate_1'
);
expect(uniqueIndex.unique).toBe(true);
expect(uniqueIndex?.unique).toBe(true);
});

it('should check if a reindex is needed', async () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ const findLabel = (value, propertyData, thesauriById, translation) => {
[]
);

const thesaurusElement = flattenedValues.find(v => v.id === value.toString());
const thesaurusElement = flattenedValues.find(v => v.id.toString() === value.toString());
if (thesaurusElement) {
const context = translation.contexts.find(
ctx => ctx.id.toString() === propertyData.content.toString()
Expand Down
Loading

0 comments on commit 4537ac5

Please sign in to comment.