diff --git a/analytics/.gitignore b/analytics/.gitignore new file mode 100644 index 000000000..49f147cb9 --- /dev/null +++ b/analytics/.gitignore @@ -0,0 +1,4 @@ + +target/ +dbt_packages/ +logs/ diff --git a/analytics/.pre-commit-config.yaml b/analytics/.pre-commit-config.yaml new file mode 100644 index 000000000..4c93b0437 --- /dev/null +++ b/analytics/.pre-commit-config.yaml @@ -0,0 +1,20 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.1 + hooks: + - id: ruff-format + args: [--config=analytics/pyproject.toml] + files: ^analytics + types_or: [python] + + - repo: https://github.com/sqlfluff/sqlfluff + rev: 3.2.5 + hooks: + - id: sqlfluff-fix + files: ^analytics + types: [sql] + additional_dependencies: + - sqlfluff-templater-dbt==3.2.5 + - dbt-postgres==1.8.2 + args: + - --config=./analytics/pyproject.toml diff --git a/analytics/.user.yml b/analytics/.user.yml new file mode 100644 index 000000000..a34e1ce86 --- /dev/null +++ b/analytics/.user.yml @@ -0,0 +1 @@ +id: bfc3b0a6-aecf-487f-8ea7-9618838206d8 diff --git a/analytics/README.md b/analytics/README.md new file mode 100644 index 000000000..794a6661b --- /dev/null +++ b/analytics/README.md @@ -0,0 +1,16 @@ +# DORA - Analytics + +### Getting started + +Try running the following commands: + +```bash +$ dbt deps +$ dbt debug +$ dbt run +$ dbt test +``` + + +### Resources: +- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) diff --git a/analytics/analyses/.gitkeep b/analytics/analyses/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/analytics/cron.json b/analytics/cron.json new file mode 100644 index 000000000..babfe5b32 --- /dev/null +++ b/analytics/cron.json @@ -0,0 +1,8 @@ +{ + "jobs": [ + { + "command": "0 4 * * * ./sync-analytics.sh", + "size": "2XL" + } + ] +} diff --git a/analytics/dbt_project.yml b/analytics/dbt_project.yml new file mode 100644 index 000000000..227cae878 --- /dev/null +++ b/analytics/dbt_project.yml @@ -0,0 +1,24 @@ +name: 'dora' +version: '1.0.0' + +profile: 'dora' + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +clean-targets: + - "target" + - "dbt_packages" + +models: + dora: + staging: + +schema: staging + +materialized: view + intermediate: + +schema: intermediate + +materialized: table diff --git a/analytics/macros/.gitkeep b/analytics/macros/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/analytics/metabase_scanner/.gitignore b/analytics/metabase_scanner/.gitignore new file mode 100644 index 000000000..ebccb5a62 --- /dev/null +++ b/analytics/metabase_scanner/.gitignore @@ -0,0 +1,2 @@ +*.egg-info +.venv diff --git a/analytics/metabase_scanner/README.md b/analytics/metabase_scanner/README.md new file mode 100644 index 000000000..0fb8695fb --- /dev/null +++ b/analytics/metabase_scanner/README.md @@ -0,0 +1,31 @@ +# metabase_scanner + +Un simple analyseur de base de métadonnées Metabase pour repérer: +- les sources les plus utilisées +- les mauvaises pratiques à éviter +- les JOINs les plus fréquents +- les questions référençant des questions +- les requêtes natives trop complexes + +A l'avenir nous pourrions aussi évaluer les index nécessaires, etc. + +## Utilisation + +Installer l'application +```bash +$ python3 -m venv .venv +$ . .venv/bin/activate +$ pip install -e . +``` + +Ouvrir un tunnel SSH vers votre base de données de métadonnées Metabase si nécessaire. + +```bash +$ ssh -L local_port: remote_address: remote_port username@server.com +$ scalingo -a app-metabase db-tunnel SCALINGO_POSTGRESQL_URL +``` + +Lancer l'application (les paramètres sont documentés à l'exécution): +```bash +$ metabase_scan 'postgres://xxx:yyy@host:port/dbname' --dashboard-ids 3 4 +``` diff --git a/analytics/metabase_scanner/metabase_scan.py b/analytics/metabase_scanner/metabase_scan.py new file mode 100644 index 000000000..03ce3269b --- /dev/null +++ b/analytics/metabase_scanner/metabase_scan.py @@ -0,0 +1,258 @@ +import re +from collections import Counter + +import sqlglot +import psycopg2 +import argparse +from urllib.parse import urlparse, unquote + +MAX_JOINS = 3 + +QUESTIONS = {} +TABLE_NAMES = {} +TABLE_FIELDS = {} + +# 73 = TDB Pilotage département +# 79 = Nouvelle stats public 2023 +# 86 = Gestion des structures ADIE +# 90 = Suivi du remplacement de la fiche de liaison - Drôme +# 92 = Objectifs 03.24 > 09.24 +# 100 = Suivi du remplacement de la fiche de liaison CD +# question 447 = Liste des gestionnaires +# question 863 = Tableau de bord biz dev MVP v4 + + +def _first_item(cur): + return [row[0] for row in cur.fetchall()] + + +def get_question_ids(cur, dashboard_id): + cur.execute( + """ + select distinct(card_id) + from report_dashboardcard + where dashboard_id = %s + """, + (dashboard_id,), + ) + return _first_item(cur) + + +def fix_metabase_syntax(query): + def replace_model_syntax(match): + placeholder = match.group(1).strip() + return f'(SELECT * FROM "{placeholder}")' + + query = re.sub(r"\[\[.*?\]\]", "", query) # Metabase optional snippets + return re.sub(r"{{([^}]+)}}", replace_model_syntax, query) + + +def extract_field_ids(lst): + """Given a Metabase filter query like: + ['and', ['=', ['field', 267, None], True], ['!=', ['field', 263, None], 'PE']] + returns a list of field IDs: [267, 263] + """ + result = [] + if isinstance(lst, list): + if len(lst) >= 3 and lst[0] == "field": + result.append(lst[1]) # Append the ID (second element) + for item in lst: + result.extend(extract_field_ids(item)) + return result + + +class MetabaseQuestion: + def __init__(self, id, query): + self.id = id + self.type = query["type"] + self.database = query["database"] + self.source_table = None + self.query = None + self.joins = [] + self.filters = [] + self.inception_level = 0 + if self.type == "native": + self.query = fix_metabase_syntax(query["native"]["query"]) + parsed = sqlglot.parse_one(self.query) + ctes = [cte.alias for cte in parsed.find_all(sqlglot.exp.CTE)] + for table in parsed.find_all(sqlglot.exp.Table): + if table.name not in ctes and not table.alias: + # first found table is considered the "source" table + if not self.source_table: + self.source_table = table.name + else: + self.joins.append(table.name) + else: + query = query["query"] + # not clear why. example: https://metabase.dora.inclusion.beta.gouv.fr/question/612 + while "source-query" in query: + query = query["source-query"] + self.source_table = query["source-table"] + if "filter" in query: + self.filters = [ + TABLE_FIELDS.get(id, id) + for id in extract_field_ids(query["filter"]) + ] + if "joins" in query: + self.joins = [join["source-table"] for join in query["joins"]] + + def resolve_inceptions(self): + if isinstance(self.source_table, str) and self.source_table.startswith( + "card__" + ): + src_question_id = int(self.source_table.removeprefix("card__")) + src_question = QUESTIONS[src_question_id] + self.source_table = src_question.source_table + self.joins += src_question.joins + self.inception_level += 1 + len(src_question.joins) + + all_joins = [] + for join in self.joins: + if isinstance(join, str) and join.startswith("card__"): + src_question_id = int(join.removeprefix("card__")) + src_question = QUESTIONS[src_question_id] + self.inception_level += 1 + len(src_question.joins) + all_joins.append(src_question.source_table) + all_joins += src_question.joins + else: + all_joins.append(join) + self.joins = all_joins + + def resolve_names(self): + self.joins = [TABLE_NAMES.get(join, join) for join in self.joins] + self.source_table = TABLE_NAMES.get(self.source_table, self.source_table) + + +def main(): + parser = argparse.ArgumentParser(description="Metabase dashboard analyzer") + parser.add_argument( + "connection_url", type=str, help="Metabase internal DB connection URL" + ) + parser.add_argument( + "--host", type=str, default="127.0.0.1", help="Tunnel host (default: 127.0.0.1)" + ) + parser.add_argument( + "--port", type=int, default=10000, help="Tunnel port (default: 10000)" + ) + parser.add_argument( + "--dashboard-ids", + nargs="+", + type=int, + default=[], + help="Space-separated numeric IDs of dashboards", + ) + parser.add_argument( + "--question-ids", + nargs="+", + type=int, + default=[], + help="Space-separated numeric IDs of questions", + ) + + args = parser.parse_args() + + parsed_url = urlparse(args.connection_url) + conn_params = { + "host": args.host, + "port": args.port, + "dbname": parsed_url.path.lstrip("/"), + "user": unquote(parsed_url.username), + "password": unquote(parsed_url.password), + } + + con = psycopg2.connect(**conn_params) + cur = con.cursor() + + dashboards = {} + + for dashboard_id in args.dashboard_ids: + question_ids = get_question_ids(cur, dashboard_id) + # some cards in a dashboard a re not a question + dashboards[dashboard_id] = [q for q in question_ids if q is not None] + + cur.execute("select id, name from metabase_table") + for id, name in cur.fetchall(): + TABLE_NAMES[id] = name + + cur.execute("select id, name from metabase_field") + for id, name in cur.fetchall(): + TABLE_FIELDS[id] = name + + cur.execute( + "select id, dataset_query::jsonb from report_card where not archived order by id" + ) + for card_id, query in cur.fetchall(): + QUESTIONS[card_id] = MetabaseQuestion(card_id, query) + + for question in QUESTIONS.values(): + try: + question.resolve_inceptions() + except KeyError: + print( + f"Could not resolve inceptions for question {question.id}, source question does not exist" + ) + continue + question.resolve_names() + + all_questions = set() + for dashboard_id, question_ids in dashboards.items(): + all_questions.update(question_ids) + print(f"############### analyse dashboard={dashboard_id}") + print("nb questions:", len(question_ids)) + all_tables = set() + print("> questions utilisées") + for question_id in sorted(question_ids): + q = QUESTIONS[question_id] + print(f"\t{q.id=} {q.source_table=} {q.inception_level=}") + for join in q.joins: + print(f"\t\tJOIN on {join}") + all_tables.add(q.source_table) + all_tables.update(q.joins) + print("> tables référencées") + for table in sorted([t for t in all_tables if t]): + print(f"\t{table}") + + for question_id in args.question_ids: + print(f"############### analyse question={question_id}") + all_questions.add(question_id) + q = QUESTIONS[question_id] + print(f"\t{q.id=} {q.source_table=} {q.inception_level=}") + for join in q.joins: + print(f"\t\tJOIN on {join}") + for filter in q.filters: + print(f"\t\tFILTER on {filter}") + + table_occurrences = Counter() + join_pairs = [] + for q_id in all_questions: + question = QUESTIONS[q_id] + for i in range(len(question.joins)): + for j in range(i + 1, len(question.joins)): + if question.joins[i] != question.joins[j]: + pair = tuple(sorted((question.joins[i], question.joins[j]))) + join_pairs.append(pair) + table_occurrences[question.source_table] += 1 + for join in question.joins: + if question.source_table != join: + table_occurrences[join] += 1 + pair = tuple(sorted((question.source_table, join))) + join_pairs.append(pair) + + print("############### top tables") + for table, count in sorted( + dict(table_occurrences).items(), key=lambda x: x[1], reverse=True + ): + print(f"{table}: {count} occurrences") + + print("############### top joins") + pair_frequencies = Counter(join_pairs) + most_joined = sorted(pair_frequencies.items(), key=lambda x: x[1], reverse=True) + for pair, freq in most_joined: + print(f"{pair[0]} - {pair[1]}: {freq}") + + cur.close() + con.close() + + +if __name__ == "__main__": + main() diff --git a/analytics/metabase_scanner/pyproject.toml b/analytics/metabase_scanner/pyproject.toml new file mode 100644 index 000000000..704b0788a --- /dev/null +++ b/analytics/metabase_scanner/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +build-backend = "setuptools.build_meta" +requires = ["setuptools", "wheel"] + +[project] +version = "0.1.0" +name = "metabase_scanner" +dependencies = [ + "psycopg2", + "sqlglot", +] + +[project.scripts] +metabase-scan = "metabase_scan:main" + +[tool.ruff.lint] +# see prefixes in https://beta.ruff.rs/docs/rules/ +select = [ + "F", # pyflakes + "E", # pycodestyle errors + "W", # pycodestyle warnings + "I", # isort + "UP", # pyupgrade +] + +[tool.ruff.lint.isort] +section-order = ["future","standard-library","third-party","first-party","local-folder"] +combine-as-imports = true diff --git a/analytics/models/_sources.yml b/analytics/models/_sources.yml new file mode 100644 index 000000000..a65510d96 --- /dev/null +++ b/analytics/models/_sources.yml @@ -0,0 +1,33 @@ +sources: + - name: dora + schema: public + tables: + - name: orientations_orientation + - name: services_fundinglabel + - name: services_savedsearch + - name: services_service + - name: services_service_categories + - name: services_servicecategory + - name: services_service_coach_orientation_modes + - name: services_service_funding_labels + - name: stats_deploymentstate + - name: stats_dimobilisationevent + - name: stats_dimobilisationevent_categories + - name: stats_diserviceview + - name: stats_mobilisationevent + - name: stats_mobilisationevent_categories + - name: stats_pageview + - name: stats_searchview + - name: stats_searchview_categories + - name: stats_serviceview + - name: stats_structureinfosview + - name: stats_structureview + - name: structures_structure + - name: structures_structuremember + - name: structures_structurenationallabel + - name: structures_structure_national_labels + - name: structures_structureputativemember + - name: structures_structuretypology + - name: users_user + + diff --git a/analytics/models/intermediate/int_mobilisationevent_user.sql b/analytics/models/intermediate/int_mobilisationevent_user.sql new file mode 100644 index 000000000..fce44bf52 --- /dev/null +++ b/analytics/models/intermediate/int_mobilisationevent_user.sql @@ -0,0 +1,17 @@ +WITH users AS ( + SELECT * FROM {{ ref('stg_user') }} +), + +events AS ( + SELECT * FROM {{ ref('stg_mobilisationevent') }} +), + +final AS ( + SELECT + {{ dbt_utils.star(relation_alias='events', from=ref('stg_mobilisationevent'), prefix='event_') }}, + {{ dbt_utils.star(relation_alias='users', from=ref('stg_user'), prefix='user_') }} + FROM events + INNER JOIN users ON events.user_id = users.id +) + +SELECT * FROM final diff --git a/analytics/models/intermediate/int_orientation_user_service.sql b/analytics/models/intermediate/int_orientation_user_service.sql new file mode 100644 index 000000000..2ff9b2f4c --- /dev/null +++ b/analytics/models/intermediate/int_orientation_user_service.sql @@ -0,0 +1,23 @@ +WITH orientations AS ( + SELECT * FROM {{ ref('stg_orientation') }} +), + +users AS ( + SELECT * FROM {{ ref('stg_user') }} +), + +services AS ( + SELECT * FROM {{ ref('int_service_structure') }} +), + +final AS ( + SELECT + {{ dbt_utils.star(relation_alias='orientations', from=ref('stg_orientation'), prefix='orientation_') }}, + {{ dbt_utils.star(relation_alias='users', from=ref('stg_user'), prefix='user_') }}, + {{ dbt_utils.star(relation_alias='services', from=ref('int_service_structure')) }} + FROM orientations + INNER JOIN users ON orientations.prescriber_id = users.id + INNER JOIN services ON orientations.service_id = services.service_id +) + +SELECT * FROM final diff --git a/analytics/models/intermediate/int_pageview_user.sql b/analytics/models/intermediate/int_pageview_user.sql new file mode 100644 index 000000000..e43b180d7 --- /dev/null +++ b/analytics/models/intermediate/int_pageview_user.sql @@ -0,0 +1,17 @@ +WITH events AS ( + SELECT * FROM {{ ref('stg_pageview') }} +), + +users AS ( + SELECT * FROM {{ ref('stg_user') }} +), + +final AS ( + SELECT + {{ dbt_utils.star(relation_alias='events', from=ref('stg_pageview'), prefix='event_') }}, + {{ dbt_utils.star(relation_alias='users', from=ref('stg_user'), prefix='user_') }} + FROM events + INNER JOIN users ON events.user_id = users.id +) + +SELECT * FROM final diff --git a/analytics/models/intermediate/int_searchview_user.sql b/analytics/models/intermediate/int_searchview_user.sql new file mode 100644 index 000000000..6cb2bdd5c --- /dev/null +++ b/analytics/models/intermediate/int_searchview_user.sql @@ -0,0 +1,17 @@ +WITH events AS ( + SELECT * FROM {{ ref('stg_searchview') }} +), + +users AS ( + SELECT * FROM {{ ref('stg_user') }} +), + +final AS ( + SELECT + {{ dbt_utils.star(relation_alias='events', from=ref('stg_searchview'), prefix='event_') }}, + {{ dbt_utils.star(relation_alias='users', from=ref('stg_user'), prefix='user_') }} + FROM events + INNER JOIN users ON events.user_id = users.id +) + +SELECT * FROM final diff --git a/analytics/models/intermediate/int_service_categories.sql b/analytics/models/intermediate/int_service_categories.sql new file mode 100644 index 000000000..0c0325e42 --- /dev/null +++ b/analytics/models/intermediate/int_service_categories.sql @@ -0,0 +1,23 @@ +WITH services AS ( + SELECT * FROM {{ ref("int_service_structure") }} +), + +categorie_pairs AS ( + SELECT * FROM {{ source('dora', 'services_service_categories') }} +), + +categories AS ( + SELECT * FROM {{ source('dora', 'services_servicecategory') }} +), + +final AS ( + SELECT + services.*, + categories.label AS category_name, + categories.value AS category_label + FROM services + LEFT JOIN categorie_pairs ON services.service_id = categorie_pairs.service_id + LEFT JOIN categories ON categorie_pairs.servicecategory_id = categories.id +) + +SELECT * FROM final diff --git a/analytics/models/intermediate/int_service_fundinglabels.sql b/analytics/models/intermediate/int_service_fundinglabels.sql new file mode 100644 index 000000000..1696f1555 --- /dev/null +++ b/analytics/models/intermediate/int_service_fundinglabels.sql @@ -0,0 +1,23 @@ +WITH services AS ( + SELECT * FROM {{ ref("int_service_structure") }} +), + +fundinglabel_pairs AS ( + SELECT * FROM {{ source('dora', 'services_service_funding_labels') }} +), + +fundinglabels AS ( + SELECT * FROM {{ source('dora', 'services_fundinglabel') }} +), + +final AS ( + SELECT + services.*, + fundinglabels.label AS fundinglabel_name, + fundinglabels.value AS fundinglabel_label + FROM services + LEFT JOIN fundinglabel_pairs ON services.service_id = fundinglabel_pairs.service_id + LEFT JOIN fundinglabels ON fundinglabel_pairs.fundinglabel_id = fundinglabels.id +) + +SELECT * FROM final diff --git a/analytics/models/intermediate/int_service_structure.sql b/analytics/models/intermediate/int_service_structure.sql new file mode 100644 index 000000000..14e0b3d37 --- /dev/null +++ b/analytics/models/intermediate/int_service_structure.sql @@ -0,0 +1,17 @@ +WITH services AS ( + SELECT * FROM {{ ref("stg_service") }} +), + +structure AS ( + SELECT * FROM {{ ref("stg_structure") }} +), + +final AS ( + SELECT + {{ dbt_utils.star(relation_alias='services', from=ref("stg_service"), prefix='service_') }}, + {{ dbt_utils.star(relation_alias='structure', from=ref("stg_structure"), prefix='structure_') }} + FROM services + INNER JOIN structure ON services.structure_id = structure.id +) + +SELECT * FROM final diff --git a/analytics/models/intermediate/int_service_structure_labels.sql b/analytics/models/intermediate/int_service_structure_labels.sql new file mode 100644 index 000000000..d932bf938 --- /dev/null +++ b/analytics/models/intermediate/int_service_structure_labels.sql @@ -0,0 +1,17 @@ +WITH services AS ( + SELECT * FROM {{ ref("stg_service") }} +), + +structure AS ( + SELECT * FROM {{ ref("int_structure_labels") }} +), + +final AS ( + SELECT + {{ dbt_utils.star(relation_alias='services', from=ref("stg_service"), prefix='service_', except=['id']) }}, + {{ dbt_utils.star(relation_alias='structure', from=ref("int_structure_labels"), prefix='structure_', except=['id']) }} + FROM services + INNER JOIN structure ON services.structure_id = structure.id +) + +SELECT * FROM final diff --git a/analytics/models/intermediate/int_serviceview_user_service.sql b/analytics/models/intermediate/int_serviceview_user_service.sql new file mode 100644 index 000000000..9ded7edc0 --- /dev/null +++ b/analytics/models/intermediate/int_serviceview_user_service.sql @@ -0,0 +1,23 @@ +WITH events AS ( + SELECT * FROM {{ ref('stg_serviceview') }} +), + +services AS ( + SELECT * FROM {{ ref('int_service_structure') }} +), + +users AS ( + SELECT * FROM {{ ref('stg_user') }} +), + +final AS ( + SELECT + {{ dbt_utils.star(relation_alias='events', from=ref('stg_serviceview'), prefix='event_') }}, + {{ dbt_utils.star(relation_alias='services', from=ref('int_service_structure')) }}, + {{ dbt_utils.star(relation_alias='users', from=ref('stg_user'), prefix='user_') }} + FROM events + INNER JOIN services ON CAST(services.service_id AS TEXT) = events.service_id + LEFT JOIN users ON events.user_id = users.id +) + +SELECT * FROM final diff --git a/analytics/models/intermediate/int_structure_labels.sql b/analytics/models/intermediate/int_structure_labels.sql new file mode 100644 index 000000000..05a747677 --- /dev/null +++ b/analytics/models/intermediate/int_structure_labels.sql @@ -0,0 +1,23 @@ +WITH structures AS ( + SELECT * FROM {{ ref("stg_structure") }} +), + +national_label_pairs AS ( + SELECT * FROM {{ source('dora', 'structures_structure_national_labels') }} +), + +national_labels AS ( + SELECT * FROM {{ source('dora', 'structures_structurenationallabel') }} +), + +final AS ( + SELECT + structures.*, + national_labels.label AS national_label_name, + national_labels.value AS national_label_code + FROM structures + LEFT JOIN national_label_pairs ON structures.id = national_label_pairs.structure_id + LEFT JOIN national_labels ON national_label_pairs.structurenationallabel_id = national_labels.id +) + +SELECT * FROM final diff --git a/analytics/models/intermediate/int_structure_member.sql b/analytics/models/intermediate/int_structure_member.sql new file mode 100644 index 000000000..62dd7b2dc --- /dev/null +++ b/analytics/models/intermediate/int_structure_member.sql @@ -0,0 +1,22 @@ +WITH users AS ( + SELECT * FROM {{ ref('stg_user') }} +), + +structure AS ( + SELECT * FROM {{ ref('stg_structure') }} +), + +member AS ( + SELECT * FROM {{ source('dora', 'structures_structuremember') }} +), + +final AS ( + SELECT + {{ dbt_utils.star(relation_alias='users', from=ref('stg_user'), prefix='user_') }}, + {{ dbt_utils.star(relation_alias='structure', from=ref('stg_structure'), prefix='structure_') }} + FROM member + INNER JOIN structure ON member.structure_id = structure.id + INNER JOIN users ON member.user_id = users.id +) + +SELECT * FROM final diff --git a/analytics/models/staging/_staging_models.yml b/analytics/models/staging/_staging_models.yml new file mode 100644 index 000000000..cd226271b --- /dev/null +++ b/analytics/models/staging/_staging_models.yml @@ -0,0 +1,88 @@ +models: + - name: stg_user + config: + indexes: + - columns: [id] + unique: true + columns: + - name: id + data_tests: + - unique + - not_null + - name: stg_structure + config: + indexes: + - columns: [id] + unique: true + columns: + - name: id + data_tests: + - unique + - not_null + + - name: stg_pageview + config: + indexes: + - columns: [id] + unique: true + columns: + - name: id + data_tests: + - unique + - not_null + - name: stg_seachview + config: + indexes: + - columns: [id] + unique: true + columns: + - name: id + data_tests: + - unique + - not_null + + - name: stg_orientation + config: + indexes: + - columns: [id] + unique: true + columns: + - name: id + data_tests: + - unique + - not_null + + - name: stg_mobilisationevent + config: + indexes: + - columns: [id] + unique: true + columns: + - name: id + data_tests: + - unique + - not_null + - name: stg_serviceview + config: + indexes: + - columns: [id] + unique: true + - columns: [structure_id] + - columns: [user_id] + - columns: [is_staff, is_manager] + columns: + - name: id + data_tests: + - unique + - not_null + + - name: stg_service + config: + indexes: + - columns: [id] + unique: true + columns: + - name: id + data_tests: + - unique + - not_null diff --git a/analytics/models/staging/stg_mobilisationevent.sql b/analytics/models/staging/stg_mobilisationevent.sql new file mode 100644 index 000000000..9529a6cc0 --- /dev/null +++ b/analytics/models/staging/stg_mobilisationevent.sql @@ -0,0 +1,60 @@ +WITH src_events AS ( + SELECT * FROM {{ source('dora', 'stats_mobilisationevent') }} +), + +src_di_events AS ( + SELECT * FROM {{ source('dora', 'stats_dimobilisationevent') }} +), + +events AS ( + SELECT + 'dora-' || src.id AS id, + src.path, + src.date, + src.anonymous_user_hash, + src.is_logged, + src.is_staff, + src.is_manager, + src.is_an_admin, + src.is_structure_admin, + src.is_structure_member, + src.user_kind, + src.structure_department, + src.user_id, + FALSE AS is_di + + FROM src_events AS src + +), + +di_events AS ( + SELECT + 'di-' || src.id AS id, + src.path, + src.date, + src.anonymous_user_hash, + src.is_logged, + src.is_staff, + src.is_manager, + src.is_an_admin, + FALSE + AS is_structure_admin, + FALSE + AS is_structure_member, + src.user_kind, + src.structure_department, + src.user_id, + TRUE AS is_di + FROM src_di_events AS src +), + + +final AS ( + SELECT * FROM events + UNION + SELECT * + FROM di_events + WHERE date >= '2024-07-01' +) + +SELECT * FROM final diff --git a/analytics/models/staging/stg_orientation.sql b/analytics/models/staging/stg_orientation.sql new file mode 100644 index 000000000..a3088cc74 --- /dev/null +++ b/analytics/models/staging/stg_orientation.sql @@ -0,0 +1,24 @@ +-- q.id=993 q.source_table='orientations_orientation' q.inception_level=0 +-- JOIN on mb_service +-- JOIN on services_service_funding_labels +-- JOIN on services_fundinglabel +-- + + +-- ou une autre table dérivées jointe avec mb_structuremember + +-- q.id=948 q.source_table='mb_serviceview_all' q.inception_level=4 +-- JOIN on structures_structuremember +-- JOIN on mb_structure +-- JOIN on mb_user + +WITH orientations AS ( + SELECT * FROM {{ source('dora', 'orientations_orientation') }} +), + +final AS ( + SELECT * FROM orientations + WHERE CAST(creation_date AS DATE) >= '2024-07-01' +) + +SELECT * FROM final diff --git a/analytics/models/staging/stg_pageview.sql b/analytics/models/staging/stg_pageview.sql new file mode 100644 index 000000000..254649533 --- /dev/null +++ b/analytics/models/staging/stg_pageview.sql @@ -0,0 +1,12 @@ +WITH events AS ( + SELECT * FROM {{ source('dora', 'stats_pageview') }} +), + +final AS ( + SELECT * FROM events + WHERE + CAST(date AS DATE) >= '2024-07-01' + AND is_staff IS FALSE +) + +SELECT * FROM final diff --git a/analytics/models/staging/stg_searchview.sql b/analytics/models/staging/stg_searchview.sql new file mode 100644 index 000000000..569b47234 --- /dev/null +++ b/analytics/models/staging/stg_searchview.sql @@ -0,0 +1,12 @@ +WITH events AS ( + SELECT * FROM {{ source('dora', 'stats_searchview') }} +), + +final AS ( + SELECT * FROM events + WHERE + CAST(date AS DATE) >= '2024-07-01' + AND is_staff IS FALSE +) + +SELECT * FROM final diff --git a/analytics/models/staging/stg_service.sql b/analytics/models/staging/stg_service.sql new file mode 100644 index 000000000..69e20f4df --- /dev/null +++ b/analytics/models/staging/stg_service.sql @@ -0,0 +1,32 @@ +WITH src AS ( + SELECT * FROM {{ source('dora', 'services_service') }} + WHERE is_model IS FALSE +), + +services AS ( + SELECT + src.{{ dbt_utils.star(relation_alias=src, from=source('dora', 'services_service'), except=['geom']) }}, + ST_Y(CAST(src.geom AS geometry)) AS latitude, + ST_X(CAST(src.geom AS geometry)) AS longitude, + CASE + WHEN + src.modification_date + '240 days' <= NOW() + AND src.status = 'PUBLISHED' + THEN 'REQUIRED' + WHEN + src.modification_date + '180 days' <= NOW() + AND src.status = 'PUBLISHED' + THEN 'NEEDED' + ELSE 'NOT_NEEDED' + END AS update_status + FROM src +), + +final AS ( + SELECT + services.*, + CONCAT('https://dora.inclusion.beta.gouv.fr/services/', services.slug) AS dora_url + FROM services +) + +SELECT * FROM final diff --git a/analytics/models/staging/stg_serviceview.sql b/analytics/models/staging/stg_serviceview.sql new file mode 100644 index 000000000..f5804e273 --- /dev/null +++ b/analytics/models/staging/stg_serviceview.sql @@ -0,0 +1,66 @@ +WITH src AS ( + SELECT * FROM {{ source('dora', 'stats_serviceview') }} +), + +src_di AS ( + SELECT * FROM {{ source('dora', 'stats_diserviceview') }} +), + +service_view AS ( + + SELECT + 'dora-' || src.id AS id, + src.path, + src.date, + src.anonymous_user_hash, + src.is_logged, + src.is_staff, + src.is_manager, + src.is_an_admin, + src.is_structure_admin, + src.is_structure_member, + src.user_kind, + src.structure_department, + src.user_id, + CAST(src.service_id AS TEXT) AS service_id, + src.service_source AS source, + src.is_orientable, + FALSE AS is_di + FROM + src +), + +di_service_view AS ( + + SELECT + 'di-' || src.id AS id, + src.path, + src.date, + src.anonymous_user_hash, + src.is_logged, + src.is_staff, + src.is_manager, + src.is_an_admin, + FALSE AS is_structure_admin, + FALSE AS is_structure_member, + src.user_kind, + src.structure_department, + src.user_id, + CAST(src.service_id AS TEXT) AS service_id, + src.source, + FALSE AS is_orientable, + TRUE AS is_di + FROM + src_di AS src +), + +final AS ( + SELECT * FROM service_view + UNION + SELECT * FROM di_service_view + WHERE + CAST(date AS DATE) >= '2024-07-01' + AND is_staff IS FALSE +) + +SELECT * FROM final diff --git a/analytics/models/staging/stg_structure.sql b/analytics/models/staging/stg_structure.sql new file mode 100644 index 000000000..62d2b445f --- /dev/null +++ b/analytics/models/staging/stg_structure.sql @@ -0,0 +1,13 @@ +WITH structures AS ( + SELECT * FROM {{ source('dora', 'structures_structure') }} +), + + +final AS ( + SELECT + structures.*, + CONCAT('https://dora.inclusion.beta.gouv.fr/structures/', structures.slug) AS dora_url + FROM structures +) + +SELECT * FROM final diff --git a/analytics/models/staging/stg_user.sql b/analytics/models/staging/stg_user.sql new file mode 100644 index 000000000..c9a82ed08 --- /dev/null +++ b/analytics/models/staging/stg_user.sql @@ -0,0 +1,30 @@ +WITH src AS ( + SELECT * FROM {{ source('dora', 'users_user') }} + WHERE + is_active IS TRUE + AND is_valid IS TRUE + AND is_staff IS FALSE +), + +final AS ( + SELECT + id, + ic_id, + email, + last_name, + first_name, + is_valid, + is_staff, + is_manager, + departments, + date_joined, + last_login, + last_service_reminder_email_sent, + newsletter, + main_activity, + departments[1] AS department, + last_service_reminder_email_sent AS last_notification_email_sent + FROM src +) + +SELECT * FROM final diff --git a/analytics/package-lock.yml b/analytics/package-lock.yml new file mode 100644 index 000000000..17c0350a1 --- /dev/null +++ b/analytics/package-lock.yml @@ -0,0 +1,4 @@ +packages: + - package: dbt-labs/dbt_utils + version: 1.3.0 +sha1_hash: 226ae69cdfbc9367e2aa2c472b01f99dbce11de0 diff --git a/analytics/packages.yml b/analytics/packages.yml new file mode 100644 index 000000000..39f82d4de --- /dev/null +++ b/analytics/packages.yml @@ -0,0 +1,3 @@ +packages: + - package: dbt-labs/dbt_utils + version: 1.3.0 diff --git a/analytics/profiles.yml b/analytics/profiles.yml new file mode 100644 index 000000000..2e4dea3b3 --- /dev/null +++ b/analytics/profiles.yml @@ -0,0 +1,15 @@ +dora: + outputs: + dev: + type: postgres + # runs on a 2XL machine + threads: 8 + host: "{{ env_var('PGHOST', 'localhost') }}" + port: "{{ env_var('PGPORT', '5432') | int }}" + user: "{{ env_var('PGUSER', 'dora') }}" + password: "{{ env_var('PGPASSWORD', 'dora') }}" + dbname: "{{ env_var('PGDATABASE', 'dora') }}" + schema: public + + target: dev + diff --git a/analytics/pyproject.toml b/analytics/pyproject.toml new file mode 100644 index 000000000..475ce70c0 --- /dev/null +++ b/analytics/pyproject.toml @@ -0,0 +1,46 @@ +[build-system] +build-backend = "setuptools.build_meta" +requires = ["setuptools", "wheel"] + +[tool.sqlfluff.core] +dialect = "postgres" +exclude_rules = [ + "layout.long_lines", + "structure.column_order", +] +templater = "dbt" + +[tool.sqlfluff.layout.type.alias_expression] +# https://docs.sqlfluff.com/en/stable/layout.html#aligned-elements +# We want non-default spacing _before_ the alias expressions. +spacing_before = "align" +# We want to align them within the next outer select clause. +# This means for example that alias expressions within the FROM +# or JOIN clause would _not_ be aligned with them. +align_within = "select_clause" +# The point at which to stop searching outward for siblings, which +# in this example would likely be the boundary of a CTE. Stopping +# when we hit brackets is usually a good rule of thumb for this +# configuration. +align_scope = "bracketed" + + +[tool.sqlfluff.rules.capitalisation.keywords] +capitalisation_policy = "upper" + +[tool.sqlfluff.rules.capitalisation.functions] +extended_capitalisation_policy = "upper" + +[tool.sqlfluff.rules.capitalisation.literals] +# Null & Boolean Literals +capitalisation_policy = "upper" + +[tool.sqlfluff.rules.convention.casting_style] +# SQL type casting +preferred_type_casting_style = "cast" + +[tool.sqlfluff.rules.references.special_chars] +# FIXME(vperron) : A retirer une fois vérifié que pas d'impact métier +# Ceci est une mauvaise pratique, les champs internes ne devraient pas +# être les champs métiers définitifs, c'est le travail de Metabase +additional_allowed_characters = "àéè '-" diff --git a/analytics/requirements.txt b/analytics/requirements.txt new file mode 100644 index 000000000..5ebbdcc29 --- /dev/null +++ b/analytics/requirements.txt @@ -0,0 +1,4 @@ +dbt-core==1.8.8 +dbt-postgres==1.8.2 +sqlfluff==3.2.5 +sqlfluff-templater-dbt==3.2.5 diff --git a/analytics/seeds/.gitkeep b/analytics/seeds/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/analytics/seeds/stats_insertion.csv b/analytics/seeds/stats_insertion.csv new file mode 100644 index 000000000..559447dbb --- /dev/null +++ b/analytics/seeds/stats_insertion.csv @@ -0,0 +1,102 @@ +Code Dept,Nom Dept,Num Accompagnateurs,Num Dea Brsa,Num Conseillers Ft Les Emplois,Num Dea 202309,Num Brsa 202309 +10,Aube,816,18 850,177,14 190,4 660 +11,Aude,1 474,23 550,234,19 430,4 120 +12,Aveyron,923,9 900,137,8 390,1 510 +13,Bouches-du-Rhône,3 993,132 230,1 460,108 620,23 610 +14,Calvados,1 968,30 800,338,25 710,5 090 +15,Cantal,395,3 820,55,3 270,550 +16,Charente,1 050,16 480,179,13 770,2 710 +17,Charente-Maritime,1 568,28 410,331,24 140,4 270 +18,Cher,1 025,14 080,146,11 140,2 940 +19,Correze,678,9 020,123,7 570,1 450 +21,Cote-d'Or,1 415,20 550,262,17 040,3 510 +22,Côtes d'Armor,1 943,24 680,282,20 720,3 960 +23,Creuse,553,5 150,70,4 080,1 070 +24,Dordogne,1 443,17 950,216,14 960,2 990 +25,Doubs,1 210,24 710,309,20 690,4 020 +26,Drôme,1 756,27 780,277,24 040,3 740 +27,Eure,1 512,26 690,274,22 690,4 000 +28,Eure-et-Loir,907,19 850,213,16 460,3 390 +29,Finistère,2 246,37 150,528,31 620,5 530 +30,Gard,1 987,50 780,402,41 160,9 620 +31,Haute-Garonne,3 419,79 000,714,66 930,12 070 +32,Gers,857,6 950,71,5 960,990 +33,Gironde,3 290,79 440,943,67 240,12 200 +34,Hérault,3 147,82 590,682,68 760,13 830 +35,Ille-et-Vilaine,2 780,43 610,507,37 890,5 720 +36,Indre,599,9 580,96,7 700,1 880 +37,Indre-et-Loire,1 499,27 670,303,23 170,4 500 +38,Isère,2 645,55 520,580,47 920,7 600 +39,Jura,697,8 510,131,7 630,880 +40,Landes,1 084,17 660,236,15 220,2 440 +41,Loir-et-Cher,1 029,12 320,115,10 650,1 670 +42,Loire,2 010,36 370,362,30 700,5 670 +43,Haute-Loire,617,8 170,97,7 120,1 050 +44,Loire-Atlantique,2 541,55 840,714,49 200,6 640 +45,Loiret,1 501,33 250,299,27 570,5 680 +46,Lot,577,8 220,70,6 610,1 610 +47,Lot-et-Garonne,812,15 430,185,12 560,2 870 +48,Lozère,287,2 170,24,1 840,330 +49,Maine-et-Loire,2 038,34 850,446,28 810,6 040 +50,Manche,1 446,16 270,238,14 030,2 240 +51,Marne,1 477,26 050,259,20 960,5 090 +52,Haute-Marne,552,6 800,109,5 480,1 320 +53,Mayenne,1 051,9 170,143,7 890,1 280 +54,Meurthe-et-Moselle,1 968,35 750,333,29 250,6 500 +55,Meuse,699,8 760,88,7 080,1 680 +56,Morbihan,1 908,28 910,352,24 920,3 990 +57,Moselle,1 733,54 510,505,44 070,10 440 +58,Nievre,1 015,7 660,105,6 250,1 410 +59,Nord,9 869,189 990,2 185,144 730,45 260 +60,Oise,1 768,41 270,482,34 760,6 510 +61,Orne,1 008,11 340,105,9 300,2 040 +62,Pas-de-Calais,5 159,78 140,993,63 310,14 830 +63,Puy-de-Dôme,1 105,29 470,291,24 000,5 470 +64,Pyrénées-Atlantiques,1 578,25 220,325,22 280,2 940 +65,Hautes-Pyrénées,942,11 270,128,9 350,1 920 +66,Pyrénées-Orientales,1 407,37 930,298,30 530,7 400 +67,Bas-Rhin,2 027,52 720,455,44 520,8 200 +68,Haut-Rhin,1 476,36 780,394,31 530,5 250 +69,Rhône,3 756,94 970,904,80 790,14 180 +70,Haute-Saone,683,9 490,100,7 970,1 520 +71,Saône-et-Loire,1 281,22 780,253,18 740,4 040 +72,Sarthe,1 326,26 110,327,21 290,4 820 +73,Savoie,1 061,17 670,236,16 130,1 540 +74,Haute-Savoie,1 333,35 450,350,32 770,2 680 +75,Paris,5 004,127 890,1 019,109 140,18 750 +76,Seine-Maritime,2 913,66 770,827,54 210,12 560 +77,Seine-et-Marne,2 191,73 830,559,62 730,11 100 +78,Yvelines,2 625,71 980,552,61 760,10 220 +79,Deux-Sèvres,1 057,12 200,193,10 570,1 630 +80,Somme,1 681,31 280,467,25 020,6 260 +81,Tarn,1 098,20 050,267,16 400,3 650 +82,Tarn-et-Garonne,823,15 020,131,11 960,3 060 +83,Var,2 073,51 460,582,43 870,7 590 +84,Vaucluse,1 649,38 390,341,31 270,7 120 +85,Vendée,1 354,22 400,347,19 980,2 420 +86,Vienne,988,16 030,213,13 570,2 460 +87,Haute-Vienne,1 004,15 500,216,12 370,3 130 +88,Vosges,1 029,18 240,250,14 860,3 380 +89,Yonne,972,14 430,163,11 920,2 510 +90,Territoire-de-Belfort,141,8 390,85,6 650,1 740 +91,Essonne,2 250,62 710,596,54 290,8 420 +92,Hauts-de-Seine,2 359,84 050,661,71 460,12 590 +93,Seine-Saint-Denis,4 374,133 490,1 158,109 950,23 540 +94,Val-de-Marne,3 273,80 220,630,68 380,11 840 +95,Val-d'Oise,2 143,75 770,592,65 040,10 730 +971,Guadeloupe,1 459,60 940,341,43 760,17 180 +972,Martinique,1 522,42 570,296,31 640,10 930 +973,Guyane,1 036,29 090,151,21 240,7 850 +974,Réunion,2 867,162 640,1 138,117 340,45 300 +976,Mayotte,1 260,23 400,36,17 400,6 000 +2A,Corse,550,4 990,64,4 530,460 +2B,Corse,550,6 690,62,5 750,940 +01,Ain,1 243,25 180,280,22 430,2 750 +02,Aisne,1 774,35 420,390,27 540,7 880 +03,Allier,1 134,17 120,164,13 700,3 420 +04,Alpes de,482,9 160,109,7 680,1 480 +05,Hautes-Alpes,403,6 450,28,5 740,710 +06,Alpes-Maritimes,2 073,57 090,512,50 210,6 880 +07,Ardèche,1 091,16 450,116,14 260,2 190 +08,Ardennes,1 051,17 750,152,13 320,4 430 +09,Ariège,451,10 230,116,7 980,2 250 diff --git a/analytics/snapshots/.gitkeep b/analytics/snapshots/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/analytics/sync-analytics.sh b/analytics/sync-analytics.sh new file mode 100755 index 000000000..cf89c6c9d --- /dev/null +++ b/analytics/sync-analytics.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# This program can also be used to synchronize the source table locally, +# you only have to provide the correct DATABASE_URL and DORA_DATABASE_URL +# with an enabled proxy to the DORA database. + +if command -v dbclient-fetcher &> /dev/null; then + dbclient-fetcher pgsql 15 +fi + +# Generate the list of tables to be exported +cat models/_sources.yml | grep ' - name' | cut -d':' -f2 > tables.txt +xargs -I {} echo -n "-t {} " < tables.txt > args.txt + +time pg_dump $DORA_DATABASE_URL --jobs=8 --format=directory --compress=1 --clean --if-exists --no-owner --no-privileges --verbose $(cat args.txt) --file=/tmp/out.dump +time psql $DATABASE_URL -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public; CREATE EXTENSION IF NOT EXISTS postgis;" +time pg_restore --dbname=$DATABASE_URL --jobs=8 --format=directory --clean --if-exists --no-owner --no-privileges --verbose /tmp/out.dump + +# Run DBT model generation & validation +cleaned_url=${DATABASE_URL#postgresql://} +cleaned_url=${cleaned_url#postgres://} +cleaned_url=${cleaned_url#postgres://} + +userpass=$(echo "$cleaned_url" | cut -d@ -f1) +export PGUSER=$(echo "$userpass" | cut -d: -f1) +export PGPASSWORD=$(echo "$userpass" | cut -d: -f2) + +hostportdb=$(echo "$cleaned_url" | cut -d@ -f2) +export PGHOST=$(echo "$hostportdb" | cut -d: -f1) +export PGPORT=$(echo "$hostportdb" | cut -d: -f2 | cut -d/ -f1) +export PGDATABASE=$(echo "$hostportdb" | cut -d/ -f2 | cut -d'?' -f1) + +dbt debug +dbt deps +dbt seed +dbt build diff --git a/analytics/tests/.gitkeep b/analytics/tests/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/back/queries/metabase/01-base/0010_mb_structure.sql b/back/queries/metabase/01-base/0010_mb_structure.sql deleted file mode 100644 index 49cd21aab..000000000 --- a/back/queries/metabase/01-base/0010_mb_structure.sql +++ /dev/null @@ -1,12 +0,0 @@ -drop table if exists mb_structure; - -create table mb_structure as -select -- noqa: AM04 - structures_structure.*, - ( - select concat('https://dora.inclusion.beta.gouv.fr/structures/', slug) - ) as dora_url -from structures_structure; - --- Keys & constraints -alter table mb_structure add primary key (id); diff --git a/back/queries/metabase/01-base/0020_mb_all_service.sql b/back/queries/metabase/01-base/0020_mb_all_service.sql deleted file mode 100644 index 76405c54b..000000000 --- a/back/queries/metabase/01-base/0020_mb_all_service.sql +++ /dev/null @@ -1,65 +0,0 @@ --- noqa: disable=LT05 -drop table if exists mb_all_service cascade; - -create table mb_all_service as -select - services_service.id, - services_service.name, - services_service.short_desc, - services_service.full_desc, - services_service.is_cumulative, - services_service.fee_details, - services_service.beneficiaries_access_modes_other, - services_service.coach_orientation_modes_other, - services_service.forms, - services_service.contact_name, - services_service.contact_phone, - services_service.contact_email, - services_service.is_contact_info_public, - services_service.remote_url, - services_service.address1, - services_service.address2, - services_service.postal_code, - services_service.city_code, - services_service.city, - services_service.recurrence, - services_service.suspension_date, - services_service.creation_date, - services_service.modification_date, - services_service.creator_id, - services_service.last_editor_id, - services_service.structure_id, - services_service.slug, - services_service.online_form, - services_service.publication_date, - services_service.diffusion_zone_details, - services_service.diffusion_zone_type, - services_service.qpv_or_zrr, - services_service.is_model, - services_service.model_id, - services_service.last_sync_checksum, - services_service.sync_checksum, - services_service.status, - services_service.moderation_date, - services_service.moderation_status, - services_service.fee_condition_id, - services_service.use_inclusion_numerique_scheme, - services_service.source_id, - services_service.data_inclusion_id, - services_service.data_inclusion_source, - (select st_y((services_service.geom)::geometry) as st_y) as latitude, - (select st_x((services_service.geom)::geometry) as st_x) as longitude, - case - when - services_service.modification_date + '240 days' <= now() - and services_service.status = 'PUBLISHED' - then 'REQUIRED' - when - services_service.modification_date + '180 days' <= now() - and services_service.status = 'PUBLISHED' - then 'NEEDED' - else 'NOT_NEEDED' - end as update_status -from services_service; - -alter table mb_all_service add primary key (id); diff --git a/back/queries/metabase/01-base/0030_mb_service.sql b/back/queries/metabase/01-base/0030_mb_service.sql deleted file mode 100644 index dedcf6f31..000000000 --- a/back/queries/metabase/01-base/0030_mb_service.sql +++ /dev/null @@ -1,9 +0,0 @@ -drop view if exists mb_service; - -create view mb_service as -select -- noqa: AM04 - *, - ( - select concat('https://dora.inclusion.beta.gouv.fr/services/', slug) - ) as dora_url -from mb_all_service where is_model is false; diff --git a/back/queries/metabase/01-base/0040_mb_model.sql b/back/queries/metabase/01-base/0040_mb_model.sql deleted file mode 100644 index 39aacdab5..000000000 --- a/back/queries/metabase/01-base/0040_mb_model.sql +++ /dev/null @@ -1,29 +0,0 @@ -drop view if exists mb_model; - -create view mb_model as -select - id, - name, - short_desc, - full_desc, - is_cumulative, - fee_details, - beneficiaries_access_modes_other, - coach_orientation_modes_other, - forms, - recurrence, - suspension_date, - creation_date, - modification_date, - creator_id, - last_editor_id, - structure_id, - slug, - online_form, - qpv_or_zrr, - sync_checksum, - fee_condition_id, - ( - select concat('https://dora.inclusion.beta.gouv.fr/modeles/', slug) - ) as dora_url -from mb_all_service where is_model is true; diff --git a/back/queries/metabase/01-base/0050_mb_service_suggestion.sql b/back/queries/metabase/01-base/0050_mb_service_suggestion.sql deleted file mode 100644 index 5d1dd862a..000000000 --- a/back/queries/metabase/01-base/0050_mb_service_suggestion.sql +++ /dev/null @@ -1,7 +0,0 @@ -drop table if exists mb_service_suggestion; - -create table mb_service_suggestion as -select service_suggestions_servicesuggestion.* -- noqa: AM04 -from service_suggestions_servicesuggestion; - -alter table mb_service_suggestion add primary key (id); diff --git a/back/queries/metabase/01-base/0060_mb_mobilisationevent_all.sql b/back/queries/metabase/01-base/0060_mb_mobilisationevent_all.sql deleted file mode 100644 index e4f1e8e21..000000000 --- a/back/queries/metabase/01-base/0060_mb_mobilisationevent_all.sql +++ /dev/null @@ -1,43 +0,0 @@ -drop table if exists mb_mobilisationevent_all; - -create table mb_mobilisationevent_all as -select - (select 'di-'::text || stats_dimobilisationevent.id) as id, - stats_dimobilisationevent.path, - stats_dimobilisationevent.date, - stats_dimobilisationevent.anonymous_user_hash, - stats_dimobilisationevent.is_logged, - stats_dimobilisationevent.is_staff, - stats_dimobilisationevent.is_manager, - stats_dimobilisationevent.is_an_admin, - (select false - ) as is_structure_admin, - (select false - ) as is_structure_member, - stats_dimobilisationevent.user_kind, - stats_dimobilisationevent.structure_department, - stats_dimobilisationevent.user_id, - (select true) as is_di -from - stats_dimobilisationevent -union -select - (select 'dora-'::text || stats_mobilisationevent.id) as id, - stats_mobilisationevent.path, - stats_mobilisationevent.date, - stats_mobilisationevent.anonymous_user_hash, - stats_mobilisationevent.is_logged, - stats_mobilisationevent.is_staff, - stats_mobilisationevent.is_manager, - stats_mobilisationevent.is_an_admin, - stats_mobilisationevent.is_structure_admin, - stats_mobilisationevent.is_structure_member, - stats_mobilisationevent.user_kind, - stats_mobilisationevent.structure_department, - stats_mobilisationevent.user_id, - (select false) as is_di -from - stats_mobilisationevent; - - -alter table mb_mobilisationevent_all add primary key (id); diff --git a/back/queries/metabase/01-base/0070_mb_mobilisationevent_categories_all.sql b/back/queries/metabase/01-base/0070_mb_mobilisationevent_categories_all.sql deleted file mode 100644 index d7a85d111..000000000 --- a/back/queries/metabase/01-base/0070_mb_mobilisationevent_categories_all.sql +++ /dev/null @@ -1,22 +0,0 @@ -drop table if exists mb_mobilisationevent_categories_all; - -create table mb_mobilisationevent_categories_all as -select - ( - select - 'di-'::text - || stats_dimobilisationevent_categories.dimobilisationevent_id - ) as mobilisationevent_id, - stats_dimobilisationevent_categories.servicecategory_id -from - stats_dimobilisationevent_categories -union -select - ( - select - 'dora-'::text - || stats_mobilisationevent_categories.mobilisationevent_id - ) as mobilisationevent_id, - stats_mobilisationevent_categories.servicecategory_id -from - stats_mobilisationevent_categories; diff --git a/back/queries/metabase/01-base/0080_mb_serviceview_all.sql b/back/queries/metabase/01-base/0080_mb_serviceview_all.sql deleted file mode 100644 index 259cb02be..000000000 --- a/back/queries/metabase/01-base/0080_mb_serviceview_all.sql +++ /dev/null @@ -1,64 +0,0 @@ -drop table if exists mb_serviceview_all; - -create table mb_serviceview_all as -select - (select 'di-'::text || disv.id) as id, - disv.path, - disv.date, - disv.anonymous_user_hash, - disv.is_logged, - disv.is_staff, - disv.is_manager, - disv.is_an_admin, - (select false) as is_structure_admin, - (select false) as is_structure_member, - disv.user_kind, - disv.structure_department, - disv.user_id, - disv.source, - (select false) as is_orientable, - (select true) as is_di, - disv.service_id::text, - disv.structure_id, - disv.service_name, - disv.structure_name -from - stats_diserviceview as disv -union -select - (select 'dora-'::text || sv.id) as id, - sv.path, - sv.date, - sv.anonymous_user_hash, - sv.is_logged, - sv.is_staff, - sv.is_manager, - sv.is_an_admin, - sv.is_structure_admin, - sv.is_structure_member, - sv.user_kind, - sv.structure_department, - sv.user_id, - sv.service_source as source, - sv.is_orientable, - (select false) as is_di, - sv.service_id::text, - sv.structure_id::text, - s.name as service_name, - st.name as structure_name -from - stats_serviceview as sv -left join services_service as s on sv.service_id = s.id -left join structures_structure as st on sv.structure_id = st.id; - -alter table mb_serviceview_all add primary key (id); - --- Indexes -create index idx_mb_serviceview_all_structure_id -on mb_serviceview_all ("structure_id"); - -create index idx_mb_serviceview_all_user_id -on mb_serviceview_all ("user_id"); - -create index idx_mb_serviceview_all_is_staff_is_manager -on mb_serviceview_all ("is_staff", "is_manager"); diff --git a/back/queries/metabase/01-base/0090_mb_serviceview_categories_all.sql b/back/queries/metabase/01-base/0090_mb_serviceview_categories_all.sql deleted file mode 100644 index 438b8858d..000000000 --- a/back/queries/metabase/01-base/0090_mb_serviceview_categories_all.sql +++ /dev/null @@ -1,15 +0,0 @@ -drop table if exists mb_serviceview_categories_all; - - -create table mb_serviceview_categories_all as -select - (select 'di-'::text || dic.diserviceview_id) as serviceview_id, - dic.servicecategory_id -from - stats_diserviceview_categories as dic -union -select - (select 'dora-'::text || svc.serviceview_id) as serviceview_id, - svc.servicecategory_id -from - stats_serviceview_categories as svc; diff --git a/back/queries/metabase/01-base/0100_mb_user.sql b/back/queries/metabase/01-base/0100_mb_user.sql deleted file mode 100644 index 1dab3252c..000000000 --- a/back/queries/metabase/01-base/0100_mb_user.sql +++ /dev/null @@ -1,27 +0,0 @@ -drop table if exists mb_user; - -create table mb_user as -select - users_user.id, - users_user.ic_id, - users_user.email, - users_user.last_name, - users_user.first_name, - users_user.is_valid, - users_user.is_staff, - users_user.is_manager, - users_user.departments, - users_user.date_joined, - users_user.last_login, - users_user.last_service_reminder_email_sent, - users_user.newsletter, - users_user.main_activity, - (select users_user.departments[1]) as department, - ( - select users_user.last_service_reminder_email_sent - ) as last_notification_email_sent --- -from users_user -where (users_user.is_active is true); - -alter table mb_user add primary key (id); diff --git a/back/queries/metabase/01-base/0110_mb_putative_members.sql b/back/queries/metabase/01-base/0110_mb_putative_members.sql deleted file mode 100644 index 1679bfef1..000000000 --- a/back/queries/metabase/01-base/0110_mb_putative_members.sql +++ /dev/null @@ -1,68 +0,0 @@ --- Utilisateurs en attente de rattachement --- (extension des données de `structures_structureputativemember`) - --- noqa: disable=LT05 - -drop table if exists mb_putative_members cascade; - -create table mb_putative_members as -select - mu.id as "ID utilisateur", - mu.first_name as "Nom", - mu.last_name as "Prénom", - mu.email as "E-mail", - mu.is_valid as "E-mail validé", - mu.date_joined as "Date de création", - mu.last_login as "Dernière connexion", - ms.name as "Nom de la structure", - ms.slug as "SLUG", - ms.dora_url as "URL Dora", - ms.department as "Département", - ss.creation_date as "Date d'invitation", - ss.is_admin as "Invitation en tant qu'admin", - -- invité : l'utilisateur ne s'est pas rattaché lui-même à la structure - ss.invited_by_admin as "Invitation par un admin", - ( - case - when mu.date_joined > now() - interval '24 months' then false - when mu.last_login is null then true - else mu.last_login < now() - interval '24 months' - end - ) as "Compte inactif", - -- TODO: un refactor avec une table utilitaire pour éviter les jointures ? - ( - select count(*) - from structures_structuremember as ssm - inner join mb_user as mu2 on ssm.user_id = mu2.id and mu2.is_valid - where ssm.is_admin and ssm.structure_id = ms.id - ) as "Nombre d'admins dans la structure", - ( - ss.is_admin - and ( - select count(*) = 0 from - structures_structuremember as ssm - inner join mb_user as mu2 on ssm.user_id = mu2.id and mu2.is_valid - where - ssm.structure_id = ss.structure_id - and ssm.is_admin - ) - ) as "Premier admin de la structure", - ( - select count(*) - from structures_structuremember as ss2 - where - ss2.user_id = mu.id - ) as "Membre d'autres structures" -from mb_user as mu -inner join structures_structureputativemember as ss on mu.id = ss.user_id -left join mb_structure as ms on ss.structure_id = ms.id -where - not mu.is_staff -order by mu.date_joined desc; - --- Indexes -create index mb_putative_members_date_joined_idx on mb_putative_members ("Date de création"); -create index mb_putative_members_dpt_idx on mb_putative_members ("Département"); -create index mb_putative_members_dpt_is_valid on mb_putative_members ("E-mail validé"); - -comment on table mb_putative_members is 'Liste des membres en attente de rattachement'; diff --git a/back/queries/metabase/01-base/0120_mb_notification_logs.sql b/back/queries/metabase/01-base/0120_mb_notification_logs.sql deleted file mode 100644 index 0e6352b10..000000000 --- a/back/queries/metabase/01-base/0120_mb_notification_logs.sql +++ /dev/null @@ -1,25 +0,0 @@ --- Historique des traitements de tâches de notification - --- noqa: disable=LT05 - -drop table if exists mb_notification_logs cascade; - -create table mb_notification_logs as -select - created_at as "date_creation", - (payload ->> 'nbCandidates')::int as "nb_candidats", - (payload ->> 'nbProcessed')::int as "nb_traitees", - (payload ->> 'nbObsolete')::int as "nb_obsoletes", - (payload ->> 'nbErrors')::int as "nb_erreurs", - payload ->> 'taskType' as "tache" -from logs_actionlog -where - msg like 'process_notification_tasks:%' - -- log level 20 => INFO - and level = 20; - --- Indexes -create index mb_notification_logs_date_creation_idx on mb_notification_logs ("date_creation"); -create index mb_notification_logs_tache_idx on mb_notification_logs ("tache"); - -comment on table mb_notification_logs is 'Historique des tâches de traitement de notification'; diff --git a/back/queries/metabase/01-base/0130_mb_stats_searchview.sql b/back/queries/metabase/01-base/0130_mb_stats_searchview.sql deleted file mode 100644 index 62814caa2..000000000 --- a/back/queries/metabase/01-base/0130_mb_stats_searchview.sql +++ /dev/null @@ -1,34 +0,0 @@ -drop table if exists mb_stats_searchview cascade; - -create table mb_stats_searchview as -select - "search".id, - "search".path, - "search".date, - "search".anonymous_user_hash, - "search".is_logged, - "search".is_staff, - "search".is_manager, - "search".is_an_admin, - "search".user_kind, - "search".department, - "search".city_code, - "search".num_results, - "search".user_id, - "search".num_di_results, - "search".num_di_results_top10, - "search".results_slugs_top10 -from stats_searchview as "search"; - --- Keys & constraints -alter table mb_stats_searchview add primary key (id); - --- Indexes -create index idx_mb_stats_searchview_is_staff_is_manager_is_logged -on mb_stats_searchview using btree ( - "is_staff", "is_manager", "is_logged" -); -create index idx_mb_stats_searchview_user_id -on mb_stats_searchview using btree ( - "user_id" -); diff --git a/back/queries/metabase/01-base/0140_mb_stats_structureview.sql b/back/queries/metabase/01-base/0140_mb_stats_structureview.sql deleted file mode 100644 index 5d00eaa3a..000000000 --- a/back/queries/metabase/01-base/0140_mb_stats_structureview.sql +++ /dev/null @@ -1,37 +0,0 @@ -drop table if exists mb_stats_structureview cascade; - -create table mb_stats_structureview as -select - "search".id, - "search".path, - "search".date, - "search".anonymous_user_hash, - "search".is_logged, - "search".is_staff, - "search".is_manager, - "search".is_an_admin, - "search".user_kind, - "search".is_structure_member, - "search".is_structure_admin, - "search".structure_department, - "search".structure_city_code, - "search".structure_id, - "search".user_id, - "search".structure_source -from stats_structureview as "search"; - --- Keys & constraints -alter table mb_stats_structureview add primary key (id); - --- Indexes -create index idx_mb_stats_structureview_date -on public.mb_stats_structureview -using btree ("date"); - -create index idx_mb_stats_structureview_filters -on public.mb_stats_structureview -using btree ("is_structure_member", "is_structure_admin", "is_staff"); - -create index idx_mb_stats_structureview_user_id -on public.mb_stats_structureview -using btree ("user_id"); diff --git a/back/queries/metabase/01-base/README.md b/back/queries/metabase/01-base/README.md deleted file mode 100644 index 05659278f..000000000 --- a/back/queries/metabase/01-base/README.md +++ /dev/null @@ -1,5 +0,0 @@ -Ici toutes les définitions de tables et vues metabase anciennement maj via scripts shell. - -Règles à avoir en tête : -* tout fichier est considéré comme une table ou vue qui sera DROP par le script -* de fait, il n'est pas possible d'avoir de fichiers avec juste de l'ajout de CONSTRAINTS ou d'INDEX ; il faut penser à reconstruire la table \ No newline at end of file diff --git a/back/queries/metabase/02-utils/README.md b/back/queries/metabase/02-utils/README.md deleted file mode 100644 index 4cd842eda..000000000 --- a/back/queries/metabase/02-utils/README.md +++ /dev/null @@ -1 +0,0 @@ -Ici les scripts SQL utilitaires / réutilisables pour la construction de questions ou requêtes Metabase. diff --git a/back/queries/metabase/03-questions/0020_q_members_invited.sql b/back/queries/metabase/03-questions/0020_q_members_invited.sql deleted file mode 100644 index 487a2f531..000000000 --- a/back/queries/metabase/03-questions/0020_q_members_invited.sql +++ /dev/null @@ -1,30 +0,0 @@ --- Question : --- liste des membres invités - --- noqa: disable=LT05 - -drop view if exists q_members_invited; - -create view q_members_invited as -select - "ID utilisateur", - "Nom", - "Prénom", - "E-mail", - "E-mail validé", - "Date de création", - "Dernière connexion", - "Nom de la structure", - "URL Dora", - "SLUG", - "Département", - "Date d'invitation", - "Invitation en tant qu'admin", - "Compte inactif", - "Nombre d'admins dans la structure", - "Premier admin de la structure", - "Membre d'autres structures" -from mb_putative_members -where "Invitation par un admin"; - -comment on view q_members_invited is 'Liste des membres invités en attente de rattachement'; diff --git a/back/queries/metabase/03-questions/0030_q_members_invited_valid.sql b/back/queries/metabase/03-questions/0030_q_members_invited_valid.sql deleted file mode 100644 index 2af1d8efc..000000000 --- a/back/queries/metabase/03-questions/0030_q_members_invited_valid.sql +++ /dev/null @@ -1,30 +0,0 @@ --- Question : --- liste des membres invités, avec e-mail validé, - --- noqa: disable=LT05 - -drop view if exists q_members_invited_valid; - -create view q_members_invited_valid as -select - "ID utilisateur", - "Nom", - "Prénom", - "E-mail", - "E-mail validé", - "Date de création", - "Dernière connexion", - "Nom de la structure", - "URL Dora", - "SLUG", - "Département", - "Date d'invitation", - "Invitation en tant qu'admin", - "Compte inactif", - "Nombre d'admins dans la structure", - "Premier admin de la structure", - "Membre d'autres structures" -from mb_putative_members -where "E-mail validé" and "Invitation par un admin"; - -comment on view q_members_invited_valid is 'Liste des membres invités en attente de rattachement : e-mail validé'; diff --git a/back/queries/metabase/03-questions/0031_q_members_invited_invalid.sql b/back/queries/metabase/03-questions/0031_q_members_invited_invalid.sql deleted file mode 100644 index 46fbfc224..000000000 --- a/back/queries/metabase/03-questions/0031_q_members_invited_invalid.sql +++ /dev/null @@ -1,30 +0,0 @@ --- Question : --- liste des membres invités, avec e-mail non-validé, - --- noqa: disable=LT05 - -drop view if exists q_members_invited_invalid; - -create view q_members_invited_invalid as -select - "ID utilisateur", - "Nom", - "Prénom", - "E-mail", - "E-mail validé", - "Date de création", - "Dernière connexion", - "Nom de la structure", - "URL Dora", - "SLUG", - "Département", - "Date d'invitation", - "Invitation en tant qu'admin", - "Compte inactif", - "Nombre d'admins dans la structure", - "Premier admin de la structure", - "Membre d'autres structures" -from mb_putative_members -where not "E-mail validé" and "Invitation par un admin"; - -comment on view q_members_invited_invalid is 'Liste des membres invités en attente de rattachement : e-mail non-validé'; diff --git a/back/queries/metabase/03-questions/0040_q_orphan_users.sql b/back/queries/metabase/03-questions/0040_q_orphan_users.sql deleted file mode 100644 index c2dda4883..000000000 --- a/back/queries/metabase/03-questions/0040_q_orphan_users.sql +++ /dev/null @@ -1,46 +0,0 @@ --- Utilisateurs sans rattachements ni invitations --- e-mail validé ou non - --- noqa: disable=LT05 - -drop table if exists q_orphan_users; - -create table q_orphan_users as -select - mu.id as "ID utilisateur", - mu.email as "E-mail", - mu.first_name as "Nom", - mu.last_name as "Prénom", - mu.is_valid as "E-mail validé", - mu.date_joined as "Date de création", - mu.last_login as "Dernière connexion", - ( - case - when mu.date_joined > now() - interval '24 months' then false - when mu.last_login is null then true - else mu.last_login < now() - interval '24 months' - end - ) as "Compte inactif", - (select ic_id is not null) as "Inscrit IC", - (select date_joined < '2022-10-03') as "Créé avant MEP IC" -from mb_user as mu -where - not mu.is_staff - and mu.id not in (select user_id from structures_structuremember) - and mu.id not in (select user_id from structures_structureputativemember) - and mu.id not in (select "ID utilisateur" from q_users_before_ic) -order by mu.date_joined desc; - --- Indexes et PK - -alter table public.q_orphan_users add constraint q_orphan_users_pk primary key ( -- noqa: LT05 - "ID utilisateur" -); - -create index q_orphan_users_valide_idx -on q_orphan_users ("E-mail validé"); - -create index q_orphan_users_date_joined_idx -on q_orphan_users ("Date de création"); - -comment on table q_orphan_users is 'Liste des utilisateurs non rattachés à une structure et sans invitation'; diff --git a/back/queries/metabase/03-questions/0050_q_users_before_ic.sql b/back/queries/metabase/03-questions/0050_q_users_before_ic.sql deleted file mode 100644 index 0d6ac030b..000000000 --- a/back/queries/metabase/03-questions/0050_q_users_before_ic.sql +++ /dev/null @@ -1,42 +0,0 @@ --- Utilisateurs sans invitations n'ayant pas validé leur email --- avant la mise en production d'inclusion-connect --- a.k.a utilisateurs à supprimer - --- noqa: disable=LT05 - -drop table if exists q_users_before_ic; - -create table q_users_before_ic as -select - mu.id as "ID utilisateur", - mu.email as "E-mail", - mu.date_joined as "Date de création", - ( - case - when mu.date_joined > now() - interval '24 months' then false - when mu.last_login is null then true - else mu.last_login < now() - interval '24 months' - end - ) as "Compte inactif", - (mu.date_joined < '2022-10-03') as "Créé avant IC" -from mb_user as mu -where - not mu.is_staff - -- adresse e-mail non validée : - and not mu.is_valid - -- aucun rattachement en attente : - and mu.id not in (select user_id from structures_structureputativemember where not invited_by_admin) - -- aucune connexion IC : - and mu.ic_id is null -order by mu.date_joined desc; - --- Indexes et PK - -alter table public.q_users_before_ic add constraint q_users_before_ic_pk primary key ( -- noqa: LT05 - "ID utilisateur" -); - -create index q_users_before_ic_date_joined_idx -on q_users_before_ic ("Date de création"); - -comment on table q_users_before_ic is 'Utilisateurs avec e-mail non validé, créés avant IC'; diff --git a/back/queries/metabase/03-questions/0060_q_searches_with_few_results.sql b/back/queries/metabase/03-questions/0060_q_searches_with_few_results.sql deleted file mode 100644 index 939cf250a..000000000 --- a/back/queries/metabase/03-questions/0060_q_searches_with_few_results.sql +++ /dev/null @@ -1,37 +0,0 @@ --- Question(s) concernée(s): --- • "Nombre de recherches aboutissant à peu de résultats (<6)" --- • "Nombre de recherches aboutissant à 0 résultat" - -drop table if exists q_searches_with_few_results; - -create table q_searches_with_few_results as ( - select - "search".id as "id", - "search".path as "path", - "search".date as "date", - "search".num_results as "num_results", - "search".department as "department", - ss.label as "label", - category.label as "category" - from stats_searchview as "search" - left join - stats_searchview_categories as "service" - on "search".id = "service".searchview_id - left join - services_servicecategory as category - on "service".servicecategory_id = category.id - left join - structures_structuremember as member - on "search".user_id = member.user_id - left join mb_structure as structure on member.structure_id = structure.id - left join - structures_structure_national_labels as ssnl - on structure.id = ssnl.structure_id - left join - structures_structurenationallabel as ss - on ssnl.structurenationallabel_id = ss.id - where - "search".num_results < 6 - and "search".is_staff = false - and "search".is_manager = false -); diff --git a/back/queries/metabase/03-questions/0070_q_searches_by_category_department_label.sql b/back/queries/metabase/03-questions/0070_q_searches_by_category_department_label.sql deleted file mode 100644 index 12074ccb0..000000000 --- a/back/queries/metabase/03-questions/0070_q_searches_by_category_department_label.sql +++ /dev/null @@ -1,36 +0,0 @@ --- Question(s) concernée(s): --- • "Nombre de recherches par thématique" - -drop table if exists q_searches_by_category_department_label; - -create table q_searches_by_category_department_label as ( - select - category.label as "category", - "search".department as "department", - ss.label as "label", - count(distinct "search".id) as "count" - from stats_searchview as "search" - left join - stats_searchview_categories as "service" - on "search".id = "service".searchview_id - left join - services_servicecategory as category - on "service".servicecategory_id = category.id - left join - structures_structuremember as member - on "search".user_id = member.user_id - left join mb_structure as structure on member.structure_id = structure.id - left join - structures_structure_national_labels as ssnl - on structure.id = ssnl.structure_id - left join - structures_structurenationallabel as ss - on ssnl.structurenationallabel_id = ss.id - where - "search".is_staff = false - and "search".is_manager = false - group by - category.label, - "search".department, - ss.label -); diff --git a/back/queries/metabase/03-questions/0080_q_searches_by_monthyear_department_label.sql b/back/queries/metabase/03-questions/0080_q_searches_by_monthyear_department_label.sql deleted file mode 100644 index 3eede3abc..000000000 --- a/back/queries/metabase/03-questions/0080_q_searches_by_monthyear_department_label.sql +++ /dev/null @@ -1,36 +0,0 @@ --- Question(s) concernée(s): --- • "Evolution du nombre de recherhces - avec typologie" - -drop table if exists q_searches_by_monthyear_department_label; - -create table q_searches_by_monthyear_department_label as ( - select - "search".department as "department", - ss.label as "label", - to_char("search".date, 'YYYY-MM') as "month_year", - count(distinct "search".id) as "count" - from stats_searchview as "search" - left join - stats_searchview_categories as "service" - on "search".id = "service".searchview_id - left join - services_servicecategory as category - on "service".servicecategory_id = category.id - left join - structures_structuremember as member - on "search".user_id = member.user_id - left join mb_structure as structure on member.structure_id = structure.id - left join - structures_structure_national_labels as ssnl - on structure.id = ssnl.structure_id - left join - structures_structurenationallabel as ss - on ssnl.structurenationallabel_id = ss.id - where - "search".is_staff = false - and "search".is_manager = false - group by - to_char("search".date, 'YYYY-MM'), - "search".department, - ss.label -); diff --git a/back/queries/metabase/03-questions/0090_q_searches_on_last_30_days.sql b/back/queries/metabase/03-questions/0090_q_searches_on_last_30_days.sql deleted file mode 100644 index 04fd7a135..000000000 --- a/back/queries/metabase/03-questions/0090_q_searches_on_last_30_days.sql +++ /dev/null @@ -1,35 +0,0 @@ --- Question(s) concernée(s): --- • "Recherches sur les 30 derniers jours" - -drop table if exists q_searches_on_last_30_days; - -create table q_searches_on_last_30_days as ( - select - "search".id as "id", - "search".path as "path", - "search".date as "date", - "search".num_results as "num_results", - "search".department as "department", - ss.label as "label" - from stats_searchview as "search" - left join - stats_searchview_categories as category - on "search".id = category.searchview_id - left join - services_servicecategory as thematique - on category.servicecategory_id = thematique.id - left join - structures_structuremember as member - on "search".user_id = member.user_id - left join mb_structure as structure on member.structure_id = structure.id - left join - structures_structure_national_labels as ssnl - on structure.id = ssnl.structure_id - left join - structures_structurenationallabel as ss - on ssnl.structurenationallabel_id = ss.id - where - "search".date >= now() - INTERVAL '30 days' - and "search".is_staff = false - and "search".is_manager = false -); diff --git a/back/queries/metabase/03-questions/0100_q_mobilisations_by_category_department.sql b/back/queries/metabase/03-questions/0100_q_mobilisations_by_category_department.sql deleted file mode 100644 index ef2a73e49..000000000 --- a/back/queries/metabase/03-questions/0100_q_mobilisations_by_category_department.sql +++ /dev/null @@ -1,36 +0,0 @@ --- Question(s) concernée(s): --- • "Nombre de mobilisations par thématique" - -drop table if exists q_mobilisations_by_category_department; - -create table q_mobilisations_by_category_department as ( - select - mobilisation.id as "id", - mobilisation.path as "path", - mobilisation.date as "date", - structure.department as "department", - ss.label as "label", - category.label as "category" - from stats_mobilisationevent as mobilisation - left join - services_service_categories as "service" - on mobilisation.service_id = "service".service_id - left join - services_servicecategory as category - on "service".servicecategory_id = category.id - left join - structures_structuremember as member - on mobilisation.user_id = member.user_id - left join mb_structure as structure on member.structure_id = structure.id - left join - structures_structure_national_labels as ssnl - on structure.id = ssnl.structure_id - left join - structures_structurenationallabel as ss - on ssnl.structurenationallabel_id = ss.id - where - mobilisation.is_staff = false - and mobilisation.is_manager = false - and mobilisation.is_structure_member = false - and mobilisation.is_structure_admin = false -); diff --git a/back/queries/metabase/03-questions/README.md b/back/queries/metabase/03-questions/README.md deleted file mode 100644 index 7f79ebc4b..000000000 --- a/back/queries/metabase/03-questions/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# Documentation : vues Postgres -À compléter au besoin ... diff --git a/back/queries/stats/01-utilisateurs/0010_v_searches_for_user.sql b/back/queries/stats/01-utilisateurs/0010_v_searches_for_user.sql deleted file mode 100644 index 11838d734..000000000 --- a/back/queries/stats/01-utilisateurs/0010_v_searches_for_user.sql +++ /dev/null @@ -1,28 +0,0 @@ --- Décompte des recherches effectuées par des utilisateurs connectés : --- non-gestionnaire, --- non-membre de l'équipe, --- et non-offreur seulement. - -drop view if exists v_searches_for_user; - -create or replace view v_searches_for_user as -select - anonymous_user_hash, - date_part('doy', date) as jour, - date_part('week', date) as semaine, - date_part('month', date) as mois, - date_part('year', date) as annee, - count(*) as nb -from mb_stats_searchview -where - user_kind != 'offreur' - and not is_manager - and not is_staff - and is_logged -group by - anonymous_user_hash, - jour, - semaine, - mois, - annee -order by annee asc, jour asc, nb desc; diff --git a/back/queries/stats/01-utilisateurs/0011_v_service_views_for_user.sql b/back/queries/stats/01-utilisateurs/0011_v_service_views_for_user.sql deleted file mode 100644 index 15bb1f5e6..000000000 --- a/back/queries/stats/01-utilisateurs/0011_v_service_views_for_user.sql +++ /dev/null @@ -1,36 +0,0 @@ --- Décompte des vues de service par utilisateur connecté : --- non-gestionnaire, --- non-membre de l'équipe, --- pas porté par une structure dont l'utilisateur est membre, --- et non-offreur seulement. - -drop view if exists v_service_views_for_user; - -create or replace view v_service_views_for_user as -select - anonymous_user_hash, - date_part('doy', date) as jour, - date_part('week', date) as semaine, - date_part('month', date) as mois, - date_part('year', date) as annee, - count(*) as nb -from stats_serviceview as ss -where - user_kind != 'offreur' - and not is_manager - and not is_staff - and is_logged - -- l'utilisateur ne visualise pas un service - -- des structures dont il est membre - and user_id not in ( - select ss2.user_id - from structures_structuremember as ss2 - where ss2.structure_id = ss.structure_id - ) -group by - anonymous_user_hash, - jour, - semaine, - mois, - annee -order by annee asc, jour asc, nb desc; diff --git a/back/queries/stats/01-utilisateurs/0012_v_structure_views_for_user.sql b/back/queries/stats/01-utilisateurs/0012_v_structure_views_for_user.sql deleted file mode 100644 index 92ef5491b..000000000 --- a/back/queries/stats/01-utilisateurs/0012_v_structure_views_for_user.sql +++ /dev/null @@ -1,35 +0,0 @@ --- Décompte des vues de structure par utilisateur connecté : --- non-gestionnaire, --- non-membre de l'équipe, --- pas une structure dont l'utilisateur est membre, --- et non-offreur seulement. - -drop view if exists v_structure_views_for_user; - -create or replace view v_structure_views_for_user as -select - anonymous_user_hash, - date_part('doy', date) as jour, - date_part('week', date) as semaine, - date_part('month', date) as mois, - date_part('year', date) as annee, - count(*) as nb -from stats_structureview as ss -where - user_kind != 'offreur' - and not is_manager - and not is_staff - and is_logged - -- l'utilisateur ne visualise pas une structure dont il est membre - and user_id not in ( - select ss2.user_id - from structures_structuremember as ss2 - where ss2.structure_id = ss.structure_id - ) -group by - anonymous_user_hash, - jour, - semaine, - mois, - annee -order by annee asc, jour asc, nb desc; diff --git a/back/queries/stats/README.md b/back/queries/stats/README.md deleted file mode 100644 index dafd4c162..000000000 --- a/back/queries/stats/README.md +++ /dev/null @@ -1 +0,0 @@ -À compléter au besoin ...