Skip to content

Commit

Permalink
Use icinga-go-library
Browse files Browse the repository at this point in the history
  • Loading branch information
lippserd committed Oct 10, 2023
1 parent 1396fd9 commit bd51c34
Show file tree
Hide file tree
Showing 66 changed files with 606 additions and 167 deletions.
6 changes: 3 additions & 3 deletions cmd/icingadb-migrate/convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@ package main
import (
"database/sql"
_ "embed"
"github.com/icinga/icingadb/pkg/database"
"github.com/icinga/icinga-go-library/database"
"github.com/icinga/icinga-go-library/types"
"github.com/icinga/icinga-go-library/utils"
icingadbTypes "github.com/icinga/icingadb/pkg/icingadb/types"
v1 "github.com/icinga/icingadb/pkg/icingadb/v1"
"github.com/icinga/icingadb/pkg/icingadb/v1/history"
"github.com/icinga/icingadb/pkg/types"
"github.com/icinga/icingadb/pkg/utils"
"github.com/jmoiron/sqlx"
"github.com/pkg/errors"
"strconv"
Expand Down
8 changes: 4 additions & 4 deletions cmd/icingadb-migrate/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@ import (
"fmt"
"github.com/creasty/defaults"
"github.com/goccy/go-yaml"
"github.com/icinga/icingadb/pkg/database"
"github.com/icinga/icinga-go-library/database"
"github.com/icinga/icinga-go-library/logging"
"github.com/icinga/icinga-go-library/types"
"github.com/icinga/icinga-go-library/utils"
"github.com/icinga/icingadb/pkg/icingadb"
"github.com/icinga/icingadb/pkg/logging"
"github.com/icinga/icingadb/pkg/types"
"github.com/icinga/icingadb/pkg/utils"
"github.com/jessevdk/go-flags"
"github.com/jmoiron/sqlx"
"github.com/jmoiron/sqlx/reflectx"
Expand Down
8 changes: 4 additions & 4 deletions cmd/icingadb-migrate/misc.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ package main
import (
"context"
"crypto/sha1"
"github.com/icinga/icingadb/pkg/database"
"github.com/icinga/icingadb/pkg/driver"
"github.com/icinga/icingadb/pkg/objectpacker"
"github.com/icinga/icingadb/pkg/types"
"github.com/icinga/icinga-go-library/database"
"github.com/icinga/icinga-go-library/driver"
"github.com/icinga/icinga-go-library/objectpacker"
"github.com/icinga/icinga-go-library/types"
"github.com/jmoiron/sqlx"
"github.com/pkg/errors"
"github.com/vbauerster/mpb/v6"
Expand Down
165 changes: 165 additions & 0 deletions cmd/icingadb/insert.go.ignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
package main

import (
"context"
"github.com/google/uuid"
"github.com/icinga/icingadb/internal/command"
"github.com/icinga/icinga-go-library/com"
"github.com/icinga/icingadb/pkg/contracts"
"github.com/icinga/icingadb/pkg/icingadb"
v1 "github.com/icinga/icingadb/pkg/icingadb/v1"
"github.com/icinga/icinga-go-library/logging"
"github.com/icinga/icinga-go-library/periodic"
"github.com/icinga/icinga-go-library/types"
"github.com/icinga/icinga-go-library/utils"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
"math/rand"
"os"
"strings"
"time"
"unsafe"
)

func main() {
os.Exit(run())
}

const count = 10000000

const schema = `DROP TABLE if EXISTS testing;
CREATE TABLE testing (
id binary(20) NOT NULL,

name varchar(255) NOT NULL,
display_name varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,

PRIMARY KEY (id),

index idx_display_name (display_name),
index idx_name (NAME)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ROW_FORMAT=DYNAMIC;`

type Testing struct {
v1.EntityWithoutChecksum
Name string `json:"name"`
DisplayName string `json:"display_name"`
}

func OnSuccessIncrement[T any](counter *com.Counter) icingadb.OnSuccess[T] {
return func(_ context.Context, rows []T) error {
counter.Add(uint64(len(rows)))
return nil
}
}

func run() int {
cmd := command.New()
logs, err := logging.NewLogging(
utils.AppName(),
cmd.Config.Logging.Level,
cmd.Config.Logging.Output,
cmd.Config.Logging.Options,
cmd.Config.Logging.Interval,
)
if err != nil {
utils.Fatal(errors.Wrap(err, "can't configure logging"))
}

logger := logs.GetLogger()
defer logger.Sync()

db, err := cmd.Database(logs.GetChildLogger("database"))
if err != nil {
logger.Fatalf("%+v", errors.Wrap(err, "can't create database connection pool from config"))
}
defer db.Close()
{
logger.Info("Connecting to database")
err := db.Ping()
if err != nil {
logger.Fatalf("%+v", errors.Wrap(err, "can't connect to database"))
}
}

ctx, cancelCtx := context.WithCancel(context.Background())
defer cancelCtx()

logger.Info("Creating schema")

for _, q := range strings.Split(schema, ";") {
q = strings.TrimSpace(q)
if q == "" {
continue
}
logger.Debugf("Performing %q", q)
if _, err := db.ExecContext(ctx, q); err != nil {
logger.Fatal(err)
}
}

logger.Info("Inserting entities")

inserts := make(chan contracts.Entity)
g, ctx := errgroup.WithContext(ctx)
g.Go(func() error {
defer close(inserts)
for i := 0; i < count; i++ {
_uuid := uuid.New()
id := types.Binary(_uuid[:])
name := RandStringBytesMaskImprSrcUnsafe(255)
inserts <- &Testing{
EntityWithoutChecksum: v1.EntityWithoutChecksum{IdMeta: v1.IdMeta{Id: id}},
Name: name,
DisplayName: RandStringBytesMaskImprSrcUnsafe(255),
}
}

return nil
})
g.Go(func() error {
var counter com.Counter

periodic.Start(ctx, time.Second, func(tick periodic.Tick) {
if count := counter.Reset(); count > 0 {
logger.Debugf("Finished %d rows", count)
}
}, periodic.OnStop(func(tick periodic.Tick) {
logger.Debugf("Finished %d rows in %s", counter.Total(), tick.Elapsed)
}))

return db.UpsertStreamed(ctx, inserts, OnSuccessIncrement[contracts.Entity](&counter))
})
if err := g.Wait(); err != nil {
logger.Fatal(err)
}

return 0
}

const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const (
letterIdxBits = 6 // 6 bits to represent a letter index
letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits
)

var src = rand.NewSource(time.Now().UnixNano())

func RandStringBytesMaskImprSrcUnsafe(n int) string {
b := make([]byte, n)
// A src.Int63() generates 63 random bits, enough for letterIdxMax characters!
for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
if remain == 0 {
cache, remain = src.Int63(), letterIdxMax
}
if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
b[i] = letterBytes[idx]
i--
}
cache >>= letterIdxBits
remain--
}

return *(*string)(unsafe.Pointer(&b))
}
29 changes: 29 additions & 0 deletions cmd/icingadb/logging.go.ignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package main

import (
"github.com/icinga/icinga-go-library/utils"
"github.com/pkg/errors"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"sync"
"time"
)

func main() {
base, err := zap.NewDevelopment()
if err != nil {
utils.Fatal(err)
}
logger := base.Sugar()
logger.Info("Starting Icinga DB Logging")
logger.Fatalf("%+v", errors.New("panic right away"))
wg := sync.WaitGroup{}
wg.Add(1)
go func() {
defer wg.Done()
<-time.After(time.Second*3)
logger.Fatalf("%+v", errors.New("panic for testing"))
}()
wg.Wait()
zapcore.WriteThenFatal
}
8 changes: 4 additions & 4 deletions cmd/icingadb/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,18 @@ import (
"context"
"fmt"
goredis "github.com/go-redis/redis/v8"
"github.com/icinga/icinga-go-library/driver"
"github.com/icinga/icinga-go-library/logging"
"github.com/icinga/icinga-go-library/redis"
"github.com/icinga/icinga-go-library/utils"
"github.com/icinga/icingadb/internal/command"
"github.com/icinga/icingadb/pkg/common"
"github.com/icinga/icingadb/pkg/driver"
"github.com/icinga/icingadb/pkg/icingadb"
"github.com/icinga/icingadb/pkg/icingadb/history"
"github.com/icinga/icingadb/pkg/icingadb/overdue"
v1 "github.com/icinga/icingadb/pkg/icingadb/v1"
"github.com/icinga/icingadb/pkg/icingaredis"
"github.com/icinga/icingadb/pkg/icingaredis/telemetry"
"github.com/icinga/icingadb/pkg/logging"
"github.com/icinga/icingadb/pkg/redis"
"github.com/icinga/icingadb/pkg/utils"
"github.com/okzk/sdnotify"
"github.com/pkg/errors"
"go.uber.org/zap"
Expand Down
Loading

0 comments on commit bd51c34

Please sign in to comment.