Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

EDDB Importer in Go #17

Draft
wants to merge 3 commits into
base: master
Choose a base branch
from
Draft

EDDB Importer in Go #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ef67525
First checkin so I can work on it everywhere
Feb 20, 2021
fa99c62
Divide and conquer
vanderaj Feb 22, 2021
70fa664
Check in
vanderaj Mar 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 191 additions & 0 deletions eddbimporter/commodities.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
package main

import (
"go.mongodb.org/mongo-driver/mongo"
)

func ProcessCommodities(client *mongo.Client) bool {
return false
}

/*
// * KodeBlox Copyright 2018 Sayak Mukhopadhyay
// *
// * Licensed under the Apache License, Version 2.0 (the "License");
// * you may not use this file except in compliance with the License.
// * You may obtain a copy of the License at
// *
// * http: //www.apache.org/licenses/LICENSE-2.0
// *
// * Unless required by applicable law or agreed to in writing, software
// * distributed under the License is distributed on an "AS IS" BASIS,
// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// * See the License for the specific language governing permissions and
// * limitations under the License.
// */

// "use strict";

// const path = require('path');
// const fs = require('fs-extra');
// const commoditiesModel = require('../../models/commodities');
// const utilities = require('../utilities');
// const eventEmmiter = require('events').EventEmitter;
// const inherits = require('util').inherits;

// let fileSize = require('../utilities/file_size');

// module.exports = Commodities;

// const pathToFile = path.resolve(__dirname, '../../dumps/listings.csv');

// function Commodities() {
// eventEmmiter.call(this);

// this.update = function () {
// let recordsUpdated = 0;
// new utilities.csvToJson(pathToFile)
// .on('start', () => {
// console.log(`EDDB commodity dump update reported`);
// this.emit('started', {
// statusCode: 200,
// update: "started",
// type: 'commodity'
// });
// })
// .on('json', json => {
// commoditiesModel
// .then(model => {
// model.findOneAndUpdate(
// { id: json.id },
// json,
// {
// upsert: true,
// runValidators: true
// })
// .then(() => {
// recordsUpdated++;
// })
// .catch((err) => {
// this.emit('error', err);
// });
// })
// .catch(err => {
// this.emit('error', err);
// });
// })
// .on('end', () => {
// console.log(`${recordsUpdated} records updated`);
// fs.unlink(pathToFile, () => {
// console.log('Commodity Dump deleted');
// });
// this.emit('done', recordsUpdated);
// })
// .on('error', err => {
// this.emit('error', err);
// })
// };

// this.import = function () {
// let recordsInserted = 0;
// new utilities.csvToJson(pathToFile)
// .on('start', () => {
// console.log(`EDDB commodity dump insertion reported`);
// this.emit('started', {
// statusCode: 200,
// insertion: "started",
// type: 'commodity'
// });
// })
// .on('json', json => {
// commoditiesModel
// .then(model => {
// let document = new model(json);
// document.save()
// .then(() => {
// recordsInserted++;
// })
// .catch((err) => {
// this.emit('error', err);
// });
// })
// .catch(err => {
// this.emit('error', err);
// });
// })
// .on('end', () => {
// console.log(`${recordsInserted} records inserted`);
// fs.unlink(pathToFile, () => {
// console.log('Commodity Dump deleted');
// });
// this.emit('done', recordsInserted);
// })
// .on('error', err => {
// this.emit('error', err);
// })
// };

// this.download = function () {
// new utilities.download('https://eddb.io/archive/v5/listings.csv', pathToFile)
// .on('start', response => {
// console.log(`EDDB commodity dump reported with status code ${response.statusCode}`);
// this.emit('started', {
// response: response,
// insertion: "started",
// type: 'commodity'
// });
// })
// .on('end', () => {
// console.log(`EDDB commodity dump saved successfully with file size ${fileSize.withPath(pathToFile)}`)
// this.emit('done');
// })
// .on('error', err => {
// this.emit('error', err);
// })
// };

// this.downloadUpdate = function () {
// let recordsUpdated = 0;
// new utilities.downloadUpdate('https://eddb.io/archive/v5/listings.csv', 'csv')
// .on('start', response => {
// console.log(`EDDB commodity dump started with status code ${response.statusCode}`);
// this.emit('started', {
// response: response,
// insertion: "started",
// type: 'commodity'
// });
// })
// .on('json', json => {
// commoditiesModel
// .then(model => {
// model.findOneAndUpdate(
// {
// id: json.id
// },
// json,
// {
// upsert: true,
// runValidators: true
// })
// .then(() => {
// recordsUpdated++;
// })
// .catch((err) => {
// this.emit('error', err);
// });
// })
// .catch(err => {
// this.emit('error', err);
// });
// })
// .on('end', () => {
// console.log(`${recordsUpdated} records updated`);
// this.emit('done', recordsUpdated);
// })
// .on('error', err => {
// this.emit('error', err);
// })
// }
// }

// inherits(Commodities, eventEmmiter);
83 changes: 83 additions & 0 deletions eddbimporter/commoditiesv6.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package main

import (
"go.mongodb.org/mongo-driver/mongo"
)

func CheckAndInsertCommoditiesV6Schema(client *mongo.Client) bool {
return false
}

// /*
// * KodeBlox Copyright 2018 Sayak Mukhopadhyay
// *
// * Licensed under the Apache License, Version 2.0 (the "License");
// * you may not use this file except in compliance with the License.
// * You may obtain a copy of the License at
// *
// * http: //www.apache.org/licenses/LICENSE-2.0
// *
// * Unless required by applicable law or agreed to in writing, software
// * distributed under the License is distributed on an "AS IS" BASIS,
// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// * See the License for the specific language governing permissions and
// * limitations under the License.
// */

// "use strict";

// const commoditiesModel = require('../../../models/v6/commodities');
// const utilities = require('../../utilities');
// const eventEmmiter = require('events').EventEmitter;
// const inherits = require('util').inherits;

// module.exports = Commodities;

// function Commodities() {
// eventEmmiter.call(this);

// this.downloadUpdate = function () {
// let recordsUpdated = 0;
// new utilities.downloadUpdate('https://eddb.io/archive/v6/listings.csv', 'csv')
// .on('start', response => {
// console.log(`EDDB commodity dump started with status code ${response.statusCode}`);
// this.emit('started', {
// response: response,
// insertion: "started",
// type: 'commodity'
// });
// })
// .on('json', json => {
// commoditiesModel
// .then(model => {
// model.findOneAndUpdate(
// {
// id: json.id
// },
// json,
// {
// upsert: true,
// runValidators: true
// })
// .then(() => {
// recordsUpdated++;
// })
// .catch((err) => {
// this.emit('error', err);
// });
// })
// .catch(err => {
// this.emit('error', err);
// });
// })
// .on('end', () => {
// console.log(`${recordsUpdated} records updated`);
// this.emit('done', recordsUpdated);
// })
// .on('error', err => {
// this.emit('error', err);
// })
// }
// }

// inherits(Commodities, eventEmmiter);
37 changes: 37 additions & 0 deletions eddbimporter/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package main

import (
"fmt"
"os"

"github.com/ilyakaznacheev/cleanenv"
)

const configFile = "config.yml"

// ConfigDatabase provides default values, which can be overriden by the configuration file and environment variables
type ConfigDatabase struct {
Collection string `yaml:"port" env:"PORT" env-default:"eddb"`
Port string `yaml:"port" env:"PORT" env-default:"27017"`
Host string `yaml:"host" env:"HOST" env-default:"localhost"`
User string `yaml:"user" env:"USER" env-default:""`
Password string `yaml:"password" env:""`
}

// Cfg is a global variable containing the configuration details
var Cfg ConfigDatabase

// ReadConfig() reads the configuration using the defaults
// The defaults can be overridden in the YAML file config.yml
// or lastly from environment variables
func ReadConfig() {
fmt.Print("Reading configuration... ")

err := cleanenv.ReadConfig(configFile, &cfg)
if err != nil {
fmt.Println(" could not read the configuration")
os.Exit(1)
}

fmt.Println("ok")
}
7 changes: 7 additions & 0 deletions eddbimporter/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
collection: "eddb"
port: 27017
host: "localhost"
user: ""
password: ""
https://eddb.io/archive/v6/bodies_recently.jsonl

40 changes: 40 additions & 0 deletions eddbimporter/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package main

import (
"context"
"fmt"
"os"
"time"

"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)

// DbClient singleton representing the database connection
// Go routines will need to clone this to make their own independant connection to the database
var DbClient *mongo.Client

// Connect to Mongo DB
func connectMongo() bool {

fmt.Print("Connecting to MongoDB... ")
connStr := "mongodb://" + cfg.User + ":" + cfg.Password + "@" + cfg.Host + ":" + cfg.Port
DbClient, err := mongo.NewClient(options.Client().ApplyURI(connStr))
if err != nil {
fmt.Print(" could not create MongoDB client")
os.Exit(1)
}

ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()

err = DbClient.Connect(ctx)
if err != nil {
fmt.Print(" could not connect to database")
os.Exit(1)
}

fmt.Println("ok")

return true
}
Loading