From 6beec54fd1119d1571cec5fdaf241d8bbcb61fe2 Mon Sep 17 00:00:00 2001 From: Dennis Priskorn Date: Sat, 4 May 2024 22:03:23 +0200 Subject: [PATCH 01/16] update gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 03f1c6e..a5e1081 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,8 @@ venv __pycache__ .mypy_cache/ config/default.py +node_modules/ +.vscode/ +config.default +package-lock.json +config/ From 3869ef32c0174c819330889dc80fcba23e8c50a8 Mon Sep 17 00:00:00 2001 From: Dennis Priskorn Date: Sat, 4 May 2024 22:03:47 +0200 Subject: [PATCH 02/16] WIP setup instructions --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..e54b665 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# OWL-MAP + +## Setup +* clone the project +* install geoip and postgres from your local package manager +* setup av venv +* enter the venv +* run 'pip install -r requirements.txt' +* cp config/examply.py config/default.py +* setup /var/lib/data/GeoLite2/GeoLite2-City.mmdb somehow +* run 'python web_view.py' From fa02c59ae784056dc6189e843fc4bfe565a73dc8 Mon Sep 17 00:00:00 2001 From: Dennis Priskorn Date: Sat, 4 May 2024 22:03:23 +0200 Subject: [PATCH 03/16] update gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 03f1c6e..a5e1081 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,8 @@ venv __pycache__ .mypy_cache/ config/default.py +node_modules/ +.vscode/ +config.default +package-lock.json +config/ From 78296ce189ff1bb78ddada2d9faf61f443932c3e Mon Sep 17 00:00:00 2001 From: Dennis Priskorn Date: Sat, 4 May 2024 22:03:47 +0200 Subject: [PATCH 04/16] WIP setup instructions --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..e54b665 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# OWL-MAP + +## Setup +* clone the project +* install geoip and postgres from your local package manager +* setup av venv +* enter the venv +* run 'pip install -r requirements.txt' +* cp config/examply.py config/default.py +* setup /var/lib/data/GeoLite2/GeoLite2-City.mmdb somehow +* run 'python web_view.py' From 99f74b12d01425e0172eddf7509231ad50c90cef Mon Sep 17 00:00:00 2001 From: Dennis Priskorn Date: Wed, 28 Aug 2024 09:42:35 +0200 Subject: [PATCH 05/16] doc: README: Add more information about postgres setup --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e54b665..5759774 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,14 @@ ## Setup * clone the project * install geoip and postgres from your local package manager -* setup av venv +* setup postgres on your machine, see e.g. https://wiki.archlinux.org/title/PostgreSQL + * `$ sudo su postgres` + * `$ initdb -D /var/lib/postgres/data` +* start postgres +* create postgres user + * `$ sudo su postgres` + * `$ createuser --interactive` -> owl +* setup a venv * enter the venv * run 'pip install -r requirements.txt' * cp config/examply.py config/default.py From 2f35dd712c31d715fd3e62e12c658f08ee60e58c Mon Sep 17 00:00:00 2001 From: Dennis Priskorn Date: Wed, 28 Aug 2024 09:50:15 +0200 Subject: [PATCH 06/16] doc: README: Add more information about venv setup --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5759774..d105c90 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,8 @@ * `$ sudo su postgres` * `$ createuser --interactive` -> owl * setup a venv -* enter the venv -* run 'pip install -r requirements.txt' + * enter the venv, e.g. in pycharm + * `$ pip install -r requirements.txt` * cp config/examply.py config/default.py * setup /var/lib/data/GeoLite2/GeoLite2-City.mmdb somehow * run 'python web_view.py' From 92456553e6341ec625a4cd1a179d83fb3146a264 Mon Sep 17 00:00:00 2001 From: Dennis Priskorn Date: Wed, 28 Aug 2024 10:44:39 +0200 Subject: [PATCH 07/16] doc: README: Add more information about download of geoip files --- README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d105c90..113c22a 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,17 @@ * create postgres user * `$ sudo su postgres` * `$ createuser --interactive` -> owl +* create the db tables * setup a venv * enter the venv, e.g. in pycharm * `$ pip install -r requirements.txt` -* cp config/examply.py config/default.py -* setup /var/lib/data/GeoLite2/GeoLite2-City.mmdb somehow +* `$ cp config/example.py config/default.py` +* download id-tagging-schema + * `$ git clone https://github.com/openstreetmap/id-tagging-schema` + * adjust paths in default.py +* download geoip databases + * https://git.io/GeoLite2-City.mmdb + * `$ pacman -S geoip-database-extra` in ArchLinux or https://dl.miyuru.lk/geoip/maxmind/city/maxmind4.dat.gz + * adjust default.py +* run the tests * run 'python web_view.py' From 55d1efaf16989c3304a23f5fb90caa2e015f9149 Mon Sep 17 00:00:00 2001 From: Dennis Priskorn Date: Sat, 4 May 2024 22:03:23 +0200 Subject: [PATCH 08/16] update gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 03f1c6e..a5e1081 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,8 @@ venv __pycache__ .mypy_cache/ config/default.py +node_modules/ +.vscode/ +config.default +package-lock.json +config/ From b191940b6e9c3ccaf9714e8707aa2419963aff60 Mon Sep 17 00:00:00 2001 From: Dennis Priskorn Date: Sat, 4 May 2024 22:03:47 +0200 Subject: [PATCH 09/16] WIP setup instructions --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..e54b665 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# OWL-MAP + +## Setup +* clone the project +* install geoip and postgres from your local package manager +* setup av venv +* enter the venv +* run 'pip install -r requirements.txt' +* cp config/examply.py config/default.py +* setup /var/lib/data/GeoLite2/GeoLite2-City.mmdb somehow +* run 'python web_view.py' From 057296012cf6326d06359cf59d4828cb1de50e52 Mon Sep 17 00:00:00 2001 From: Dennis Priskorn Date: Wed, 28 Aug 2024 09:42:35 +0200 Subject: [PATCH 10/16] doc: README: Add more information about postgres setup --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e54b665..5759774 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,14 @@ ## Setup * clone the project * install geoip and postgres from your local package manager -* setup av venv +* setup postgres on your machine, see e.g. https://wiki.archlinux.org/title/PostgreSQL + * `$ sudo su postgres` + * `$ initdb -D /var/lib/postgres/data` +* start postgres +* create postgres user + * `$ sudo su postgres` + * `$ createuser --interactive` -> owl +* setup a venv * enter the venv * run 'pip install -r requirements.txt' * cp config/examply.py config/default.py From 47104df2a79526e0e99a48410880d1bf224ec18e Mon Sep 17 00:00:00 2001 From: Dennis Priskorn Date: Wed, 28 Aug 2024 09:50:15 +0200 Subject: [PATCH 11/16] doc: README: Add more information about venv setup --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5759774..d105c90 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,8 @@ * `$ sudo su postgres` * `$ createuser --interactive` -> owl * setup a venv -* enter the venv -* run 'pip install -r requirements.txt' + * enter the venv, e.g. in pycharm + * `$ pip install -r requirements.txt` * cp config/examply.py config/default.py * setup /var/lib/data/GeoLite2/GeoLite2-City.mmdb somehow * run 'python web_view.py' From c4dfa48fa3df0a7e0dea593d38c4de5d469999a5 Mon Sep 17 00:00:00 2001 From: Dennis Priskorn Date: Wed, 28 Aug 2024 10:44:39 +0200 Subject: [PATCH 12/16] doc: README: Add more information about download of geoip files --- README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d105c90..113c22a 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,17 @@ * create postgres user * `$ sudo su postgres` * `$ createuser --interactive` -> owl +* create the db tables * setup a venv * enter the venv, e.g. in pycharm * `$ pip install -r requirements.txt` -* cp config/examply.py config/default.py -* setup /var/lib/data/GeoLite2/GeoLite2-City.mmdb somehow +* `$ cp config/example.py config/default.py` +* download id-tagging-schema + * `$ git clone https://github.com/openstreetmap/id-tagging-schema` + * adjust paths in default.py +* download geoip databases + * https://git.io/GeoLite2-City.mmdb + * `$ pacman -S geoip-database-extra` in ArchLinux or https://dl.miyuru.lk/geoip/maxmind/city/maxmind4.dat.gz + * adjust default.py +* run the tests * run 'python web_view.py' From 9f90fbd52de6857d9d634c9d63674f7aaec6e07c Mon Sep 17 00:00:00 2001 From: Dennis Priskorn Date: Wed, 28 Aug 2024 11:34:35 +0200 Subject: [PATCH 13/16] doc: README: Add more information about setup of database and role --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 113c22a..f091533 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,15 @@ * start postgres * create postgres user * `$ sudo su postgres` - * `$ createuser --interactive` -> owl + * `$ createuser --interactive` -> "owl" role with relevant priviledges +* create postgres database "matcher" + * `$ psql -U postgres` + * `> CREATE DATABASE matcher;` + * `> ALTER DATABASE matcher OWNER TO owl;` * create the db tables + * TODO find out where credentials are stored + * setup credentials in default.py + * `$ python create_db.py` * setup a venv * enter the venv, e.g. in pycharm * `$ pip install -r requirements.txt` From 8dae88d6b4f1642b6032ca3fdf5de1c23e5ed2e1 Mon Sep 17 00:00:00 2001 From: Dennis Priskorn Date: Wed, 28 Aug 2024 11:47:01 +0200 Subject: [PATCH 14/16] fix: setup: Add DB_USER and DB_NAME variables to example config and use them when creating the tables. --- config/example.py | 3 +++ create_db.py | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config/example.py b/config/example.py index 28f5052..26b8a0e 100644 --- a/config/example.py +++ b/config/example.py @@ -1,5 +1,8 @@ """Sample config.""" +# POSTGIS +DB_USER = "owl" +DB_NAME = "matcher" ID_TAGGING_SCHEMA_DIR = "/var/lib/data/id-tagging-schema" ID_PRESET_DIR = "/var/lib/data/id-tagging-schema/data/presets/" diff --git a/create_db.py b/create_db.py index 2c1c6f9..761fce7 100755 --- a/create_db.py +++ b/create_db.py @@ -2,9 +2,10 @@ from sqlalchemy.schema import CreateIndex, CreateTable +from config.default import DB_USER, DB_NAME from matcher import database, model -DB_URL = "postgresql:///matcher" +DB_URL = f"postgresql://{DB_USER}@localhost/{DB_NAME}" database.init_db(DB_URL) From 171d13f9ca4c5b4e247a0f4325a2da296f6b39f9 Mon Sep 17 00:00:00 2001 From: Dennis Priskorn Date: Wed, 28 Aug 2024 11:47:21 +0200 Subject: [PATCH 15/16] doc: README: Add more information about creating the database and tables --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f091533..6a5bec6 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ ## Setup * clone the project -* install geoip and postgres from your local package manager +* install geoip and postgis from your local package manager + * `$ sudo pacman -S postgis geoip` * setup postgres on your machine, see e.g. https://wiki.archlinux.org/title/PostgreSQL * `$ sudo su postgres` * `$ initdb -D /var/lib/postgres/data` @@ -10,13 +11,14 @@ * create postgres user * `$ sudo su postgres` * `$ createuser --interactive` -> "owl" role with relevant priviledges -* create postgres database "matcher" +* create postgres database "matcher" with hstore and postgis extensions * `$ psql -U postgres` * `> CREATE DATABASE matcher;` * `> ALTER DATABASE matcher OWNER TO owl;` + * `> CREATE EXTENSION hstore;` + * `> CREATE EXTENSION postgis;` * create the db tables - * TODO find out where credentials are stored - * setup credentials in default.py + * setup database credentials and name in default.py * `$ python create_db.py` * setup a venv * enter the venv, e.g. in pycharm From 8990daec21897dcb5e66f95ea07044e86a426308 Mon Sep 17 00:00:00 2001 From: Dennis Priskorn Date: Wed, 28 Aug 2024 11:54:05 +0200 Subject: [PATCH 16/16] chore: pytest: Add pytest and enclose the tests in a new class TestUtils --- requirements.txt | 1 + tests/test_utils.py | 41 ++++++++++++++++++++--------------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1a601c3..5e48821 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ simplejson user_agents num2words psycopg2 +pytest===8.3.2 \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py index c9f6c7f..823678e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,30 +1,29 @@ """Test matcher utils.""" +from unittest import TestCase from matcher import utils -def test_format_wikibase_time_year() -> None: - """Test passing a year to format_wikibase_time.""" - v = {"time": "+1950-00-00T00:00:00Z", "precision": 9} - assert utils.format_wikibase_time(v) == "1950" +class TestUtils(TestCase): + def test_format_wikibase_time_year(self) -> None: + """Test passing a year to format_wikibase_time.""" + v = {"time": "+1950-00-00T00:00:00Z", "precision": 9} + assert utils.format_wikibase_time(v) == "1950" + def test_format_wikibase_time_century(self) -> None: + """Test passing centuries to format_wikibase_time.""" + v = {"time": "+0800-00-00T00:00:00Z", "precision": 7} + assert utils.format_wikibase_time(v) == "8th century" -def test_format_wikibase_time_century() -> None: - """Test passing centuries to format_wikibase_time.""" - v = {"time": "+0800-00-00T00:00:00Z", "precision": 7} - assert utils.format_wikibase_time(v) == "8th century" + v = {"time": "+1950-00-00T00:00:00Z", "precision": 7} + assert utils.format_wikibase_time(v) == "20th century" - v = {"time": "+1950-00-00T00:00:00Z", "precision": 7} - assert utils.format_wikibase_time(v) == "20th century" + def test_format_wikibase_time_decade(self) -> None: + """Test passing a full date to format_wikibase_time.""" + v = {"time": "+1910-00-00T00:00:00Z", "precision": 8} + assert utils.format_wikibase_time(v) == "1910s" - -def test_format_wikibase_time_decade() -> None: - """Test passing a full date to format_wikibase_time.""" - v = {"time": "+1910-00-00T00:00:00Z", "precision": 8} - assert utils.format_wikibase_time(v) == "1910s" - - -def test_format_wikibase_time_day() -> None: - """Test passing a full date to format_wikibase_time.""" - v = {"time": "+1868-01-09T00:00:00Z", "precision": 11} - assert utils.format_wikibase_time(v) == "9 January 1868" + def test_format_wikibase_time_day(self) -> None: + """Test passing a full date to format_wikibase_time.""" + v = {"time": "+1868-01-09T00:00:00Z", "precision": 11} + assert utils.format_wikibase_time(v) == "9 January 1868"