save

ObserveRTC · Jan 10, 2025 · 3edb2a1 · 3edb2a1
1 parent 411cc00
commit 3edb2a1
Show file tree

Hide file tree

Showing 160 changed files with 14,584 additions and 40,166 deletions.
diff --git a/ProtobufConverterV3.js b/ProtobufConverterV3.js
@@ -110,7 +110,7 @@ export class ProtobufConverterV3 {
             return;
         }
         let protoType = undefined;
-        if (this._uuidFields.has(field.name)) {
+        if (this._uuidFields.has(field.name) || field.name === "appData") {
             protoType = "bytes";
         } else {
             protoType = this._mapPrimitive(type);

diff --git a/README.md b/README.md
@@ -7,51 +7,13 @@ Schemas describe the communication between services in the ObserveRTC stack. The
 * [CSV](csv-headers/) when exporting reports in CSV format from the observer, the order of the columns might be important.
 
 
-# Install
-
-## NPM
-```javascript
-npm i @observertc/schemas
-```
-
-
 # Versioning
 Schemas use SemVer version numbers of `MAJOR`.`MINOR`.`PATCH`.
 
 Increasing the number of `PATCH`, `MINOR`, or `MAJOR` implies the following:
 
  * `PATCH` indicates a change in the related libraries (e.g., bugfix in encoder/decoder).
- * `MINOR` the source schema has been altered.
+ * `MINOR` Represents the day of the WebRTC Stats draft published the schema is based on (e.g.: 20241107)
  * `MAJOR` implies conceptual changes in the schema.
 
-## Change the schema
-
-### Schema change requests
-
-Depending on which minor (or major) version is coming, you can write it in the [discussion](https://github.com/ObserveRTC/schemas/discussions).
-
-
-## Create PR for schema changes
-
-Discussion is good to request a schema, PR is better.
-Step to create a PR (after you cloned the repo):
-1. create your branch
-2. change the [source of the schemas](/sources), `sources/version.txt`, trace the change in the `sources/CHANGELOG.md`.
-3. Generate `npm-lib` typescripts (see below)
-4. open the PR
-
-
-
-
-# Develop
-
-To run the schema generator for npm-lib:
-
-```javascript
-  git clone https://github.com/observertc/schemas && \
-  cd schemas && \
-  npm i && \
-  node index.js
-```
-
 
diff --git a/index.js b/index.js
@@ -24,7 +24,6 @@ const CSV_OUTPUTS_PATH = `./${OUTPUTS_PATH}/csv`;
 const SQL_OUTPUTS_PATH = `./${OUTPUTS_PATH}/sql`;
 
 const NPM_SAMPLES_LIB_PATH = "./npm-samples-lib";
-const NPM_REPORTS_LIB_PATH = "./npm-reports-lib";
 const W3C_STATS_IDENTIFIERS = "./sources/w3c/W3cStatsIdentifiers.ts";
 
 async function rmDir(folderPath) {
@@ -44,7 +43,7 @@ async function rmDir(folderPath) {
 }
 
 function fetchChunks() {
-    for (const schemaType of ["reports", "samples"]) {
+    for (const schemaType of ["samples"]) {
         const schemaPath = path.join(SOURCE_PATH, schemaType);
         for (const file of fs.readdirSync(schemaPath)) {
             if (!file.endsWith(".chunk.avsc")) continue;
@@ -60,7 +59,7 @@ function fetchChunks() {
 
 function fetchSources() {
     const sources = new Map();
-    for (const schemaType of ["reports", "samples"]) {
+    for (const schemaType of ["samples"]) {
         const schemaPath = path.join(SOURCE_PATH, schemaType);
         for (const file of fs.readdirSync(schemaPath)) {
             if (!file.endsWith("avsc")) continue;
@@ -94,7 +93,6 @@ const main = async () => {
     const sources = fetchSources();
     const w3cStatsIdentifiers = fs.readFileSync(W3C_STATS_IDENTIFIERS, 'utf-8');
     const npmSamplesLib = new NpmLib(NPM_SAMPLES_LIB_PATH);
-    const npmReportsLib = new NpmLib(NPM_REPORTS_LIB_PATH);
     const npmEncoderLib = new NpmSampleEncoderLib();
     const npmDecoderLib = new NpmSamplesDecoderLib();
     const version = fs.readFileSync(path.join(SOURCE_PATH, "version.txt"), 'utf-8');
@@ -103,13 +101,9 @@ const main = async () => {
     const markdownLists = [];
     const bigQueryTables = [];
     const redshiftTables = [];
-    const reportTypes = [
-        `/**\n * Schema Version: ${version} \n */`
-    ];
     for (const [fileName, source] of sources) {
         const avsc = source.getAvsc();
         const schemaType = source.getSchemaType();
-        const isReport = fileName.includes("report");
         let schema;
         try {
             schema = JSON.parse(avsc);
@@ -133,71 +127,40 @@ const main = async () => {
 
         fs.writeFileSync(path.join(TYPESCRIPT_OUTPUTS_PATH, `${schemaName}.ts`), module);
 
-        if (isReport) {
-            const { csvColumnList, createTable } = makeRedshiftSql(schema);
-            const { createTable: createBigqueryTable } = makeBigQuerySql(schema);
-            if (csvColumnList) {
-                fs.writeFileSync(path.join(CSV_OUTPUTS_PATH, `${schemaName}-csv-header.txt`), csvColumnList);
-            } else {
-                console.warn(`No csv header txt is generated for ${schemaName}`);
-            }
-
-            if (createTable) {
-                redshiftTables.push(createTable);
-                // fs.writeFileSync(path.join(SQL_OUTPUTS_PATH, `${schemaName}-redshift.sql`), createTable);
-            } else {
-                console.warn(`No redshift sql is generated for ${schemaName}`);
-            }
-            if (createBigqueryTable) {
-                bigQueryTables.push(createBigqueryTable);
-                // fs.writeFileSync(path.join(SQL_OUTPUTS_PATH, `${schemaName}-bigquery.sql`), createBigqueryTable);
-            } else {
-                console.warn(`No bigquery sql is generated for ${schemaName}`);
-            }
-            reportTypes.push(module);
-            npmReportsLib.addEntry({
-                fileName,
-                schemaName,
-                schemaType,
-                exports,
-                typescript: module,
-                markdown: markdownDoc,
-            });
-        } else {
+        if (fileName === "ClientSample") {
             npmDecoderLib.addSamplesTsCode(module);
             npmEncoderLib.addSamplesTsCode(module);
-            npmSamplesLib.addEntry({
-                fileName,
-                schemaName,
-                schemaType,
-                exports,
-                typescript: module,
-                markdown: markdownDoc,
-            });
         }
+        npmSamplesLib.addEntry({
+            fileName,
+            schemaName,
+            schemaType,
+            exports,
+            typescript: module,
+            markdown: markdownDoc,
+        });
+
         fs.writeFileSync(path.join(AVSC_OUTPUTS_PATH, `${schemaName}.avsc`), avsc);
     }
     fs.writeFileSync(`schemaList.md`, markdownLists.join(`\n`))
     fs.writeFileSync(path.join(SQL_OUTPUTS_PATH, `bigquery.sql`), bigQueryTables.join("\n\n"));
     fs.writeFileSync(path.join(SQL_OUTPUTS_PATH, `redshift.sql`), redshiftTables.join("\n\n"));
-    fs.writeFileSync(path.join(TYPESCRIPT_OUTPUTS_PATH, `ReportTypes.ts`), reportTypes.join("\n"));
 
     // generate protobuf schema if we can
-
-    const samplesSource = sources.get("samples");
-    if (samplesSource) {
-        const schema = JSON.parse(samplesSource.getAvsc());
+    const clientSampleSource = sources.get("ClientSample");
+    if (clientSampleSource) {
+        const schema = JSON.parse(clientSampleSource.getAvsc());
         const protobufSchema = protobufUtils.convertToProtobufSchema(schema, version);
-        fs.writeFileSync(path.join(PROTO_OUTPUTS_PATH, "ProtobufSamples.proto"), protobufSchema);
+        fs.writeFileSync(path.join(PROTO_OUTPUTS_PATH, "ProtobufClientSample.proto"), protobufSchema);
 
         const protobufSchemaV3 = protobufUtils.convertToProtobufSchemaV3(schema, version);
-        fs.writeFileSync(path.join(PROTO_OUTPUTS_PATH, "ProtobufSamplesV3.proto"), protobufSchemaV3);
+        fs.writeFileSync(path.join(PROTO_OUTPUTS_PATH, "ProtobufClientSampleV3.proto"), protobufSchemaV3);
 
         const protobufSchemaV3Optional = protobufUtils.convertToProtobufSchemaV3(schema, version, true);
-        const v3schemaOptionalPath = path.join(PROTO_OUTPUTS_PATH, "ProtobufSamplesV3Optional.proto");
+        const v3schemaOptionalPath = path.join(PROTO_OUTPUTS_PATH, "ProtobufClientSampleV3Optional.proto");
         fs.writeFileSync(v3schemaOptionalPath, protobufSchemaV3Optional);
         await protobufUtils.createTypescriptModels(v3schemaOptionalPath, path.join(TEMP_PATH));
-        const protobufSchemaV3OptionalTs = fs.readFileSync(path.join(TEMP_PATH, "outputs", "proto", "ProtobufSamplesV3Optional_pb.ts"), 'utf-8');
+        const protobufSchemaV3OptionalTs = fs.readFileSync(path.join(TEMP_PATH, "outputs", "proto", "ProtobufClientSampleV3Optional_pb.ts"), 'utf-8');
         npmEncoderLib.addSamplesProtobufTsCode(protobufSchemaV3OptionalTs);
         npmDecoderLib.addSamplesProtobufTsCode(protobufSchemaV3OptionalTs);
         // console.log(protobufSchemaV3OptionalTs);
@@ -207,20 +170,16 @@ const main = async () => {
 
     const changelog = fs.readFileSync(path.join(SOURCE_PATH, "CHANGELOG.md"), 'utf-8');
     npmSamplesLib.version = version;
-    npmReportsLib.version = version;
     npmEncoderLib.version = version;
     npmDecoderLib.version = version;
 
     npmSamplesLib.changelog = changelog;
-    npmReportsLib.changelog = changelog;
 
     npmSamplesLib.clear();
-    npmReportsLib.clear();
     npmEncoderLib.clear();
     npmDecoderLib.clear();
 
     npmSamplesLib.make();
-    npmReportsLib.make();
     npmEncoderLib.make();
     npmDecoderLib.make();
 

diff --git a/makeTsModule.js b/makeTsModule.js
@@ -88,6 +88,15 @@ function getTsType(avroType, addDoc = true) {
         tsObj,
         tsDependencies,
     }
+
+}
+
+function getAppDataTsType() {
+    return {
+        tsType: "Record<string, unknown>",
+        tsObj: undefined,
+        tsDependencies: undefined,
+    }
 }
 
 function makeTsObj(avroSchema, addDoc = true) {
@@ -103,7 +112,7 @@ function makeTsObj(avroSchema, addDoc = true) {
         const tsTypes = [];
         for (const avroType of avroTypes) {
             if (avroType === "null") continue;
-            const { tsType, tsObj, tsDependencies } = getTsType(avroType, addDoc);
+            const { tsType, tsObj, tsDependencies } = field.name === 'appData' ? getAppDataTsType() : getTsType(avroType, addDoc);
             if (tsObj) {
                 dependencies.push(tsObj);
                 if (tsDependencies) {

diff --git a/npm-reports-lib/.gitignore b/npm-reports-lib/.gitignore
diff --git a/npm-reports-lib/.npmrc b/npm-reports-lib/.npmrc