commit 78f966600505482396232988e627614d61d969b1 Author: dresber Date: Thu Jan 6 12:20:11 2022 +0100 initial diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..af7a5d2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/data/ +/config/ +/MOP_v9-9-9.zip +.idea/ \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..de97e25 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +altgraph==0.17.2 +future==0.18.2 +invoke==1.6.0 +pefile==2021.9.3 +pyinstaller==4.7 +pyinstaller-hooks-contrib==2021.4 +pywin32-ctypes==0.2.0 +ruamel.yaml==0.17.20 +ruamel.yaml.clib==0.2.6 diff --git a/src/data_exporter.py b/src/data_exporter.py new file mode 100644 index 0000000..ce33b5a --- /dev/null +++ b/src/data_exporter.py @@ -0,0 +1,62 @@ +""" +""" + +# --------------------------------------- # +# imports # +# --------------------------------------- # +from csv import DictWriter, QUOTE_NONNUMERIC + + +# --------------------------------------- # +# definitions # +# --------------------------------------- # + + +# --------------------------------------- # +# global vars # +# --------------------------------------- # + + +# --------------------------------------- # +# functions # +# --------------------------------------- # +def _get_all_columns_from_orders(order_data): + columns = [] + + for buyer, orders in order_data.items(): + for order_data, order in orders.items(): + for order_key, order_value in order.items(): + if order_key not in columns: + columns.append(order_key) + + return columns + + +def _fill_order_with_missing_columns(order, required_columns): + for required_column in required_columns: + if required_column not in order: + order[required_column] = "" + + return order + + +def export_order_to_csv(order_data, export_file_path): + seen_columns = _get_all_columns_from_orders(order_data) + + with open(export_file_path, 'w', newline='\n') as csvfile: + writer = DictWriter(csvfile, fieldnames=seen_columns, delimiter=";") + writer.writeheader() + + for buyer, orders in order_data.items(): + for order_data, order in orders.items(): + writer.writerow(_fill_order_with_missing_columns(order, seen_columns)) + + +# --------------------------------------- # +# classes # +# --------------------------------------- # + + +# --------------------------------------- # +# main # +# --------------------------------------- # diff --git a/src/mail_parser.py b/src/mail_parser.py new file mode 100644 index 0000000..b73c3bc --- /dev/null +++ b/src/mail_parser.py @@ -0,0 +1,64 @@ +""" +""" + +# --------------------------------------- # +# imports # +# --------------------------------------- # +import re + + +# --------------------------------------- # +# definitions # +# --------------------------------------- # +START_LINE = "-------------------------------------\n" +ADDITIONAL_SECTION = "Sonstiges (nach Verfügbarkeit)" +PHONE_NUMBER_SECTION = "Telefonnummer" + + +# --------------------------------------- # +# global vars # +# --------------------------------------- # + + +# --------------------------------------- # +# functions # +# --------------------------------------- # +def parse_mail_and_return_order(order_lines): + order = {} + start_found = False + + for line in order_lines: + if start_found: + if line != "\n": + split_line = line.split(":") + + if len(split_line) == 2: + key = split_line[0] + value = split_line[1].replace("\n", "") + elif "Datum/Uhrzeit" in split_line[0]: + key = "Datum/Uhrzeit" + value = re.search(r'(?<=Uhrzeit: ).*', line)[0] + + if key != ADDITIONAL_SECTION: + if key == PHONE_NUMBER_SECTION: + order[key] = '"{}"'.format(value.strip()) + else: + order[key] = value.strip() + else: + if value != "": + for option in re.sub(r'\([^)]*\)', '', value).split(","): + order["Sonstiges {}".format(option)] = "ja" + else: + if line == START_LINE: + start_found = True + + return order + +# --------------------------------------- # +# classes # +# --------------------------------------- # + + +# --------------------------------------- # +# main # +# --------------------------------------- # diff --git a/src/start_app.py b/src/start_app.py new file mode 100644 index 0000000..55dfca1 --- /dev/null +++ b/src/start_app.py @@ -0,0 +1,77 @@ +""" +""" + +# --------------------------------------- # +# imports # +# --------------------------------------- # +import os + +from pathlib import Path + +from ruamel.yaml import YAML + +from data_exporter import export_order_to_csv +from mail_parser import parse_mail_and_return_order + +# --------------------------------------- # +# definitions # +# --------------------------------------- # +ORDER_FILE = "../config/existing_orders.yml" +DATA_IMPORT_DIRECTORY = "../data/" +EXPORT_FILE = "../data/Bestellungen.csv" + +# --------------------------------------- # +# global vars # +# --------------------------------------- # + + +# --------------------------------------- # +# functions # +# --------------------------------------- # +def _check_if_directory_exists_for_correct_run(): + if not os.path.exists("../data/"): + os.mkdir("../data/") + + if not os.path.exists("../config/"): + os.mkdir("../config/") + +# --------------------------------------- # +# classes # +# --------------------------------------- # + + +# --------------------------------------- # +# main # +# --------------------------------------- # +if __name__ == "__main__": + _check_if_directory_exists_for_correct_run() + + if os.path.exists(ORDER_FILE): + order_path = Path(ORDER_FILE) + order_data = YAML(typ='safe') + order_data.default_flow_style = False + existing_orders = order_data.load(order_path) + else: + existing_orders = {} + + for filename in os.listdir(DATA_IMPORT_DIRECTORY): + f = os.path.join(DATA_IMPORT_DIRECTORY, filename) + if filename != EXPORT_FILE and filename.endswith(".txt"): + if os.path.isfile(f): + with open(f, "r", encoding="utf-8") as order_file: + parsed_order = parse_mail_and_return_order(order_file.readlines()) + + if parsed_order["Name"] in existing_orders: + if parsed_order["Datum/Uhrzeit"] in existing_orders[parsed_order["Name"]]: + print("Order for {} from {} already added!".format(parsed_order["Name"], parsed_order["Datum/Uhrzeit"])) + else: + existing_orders[parsed_order["Name"]][parsed_order["Datum/Uhrzeit"]] = parsed_order + else: + existing_orders[parsed_order["Name"]] = {parsed_order["Datum/Uhrzeit"]: parsed_order} + + export_order_to_csv(existing_orders, EXPORT_FILE) + + with open(ORDER_FILE, "w", encoding="utf-8") as order_file: + order_data = YAML(typ='safe') + order_data.default_flow_style = False + order_data.dump(existing_orders, order_file) diff --git a/start_app.spec b/start_app.spec new file mode 100644 index 0000000..e115506 --- /dev/null +++ b/start_app.spec @@ -0,0 +1,40 @@ +# -*- mode: python ; coding: utf-8 -*- + + +block_cipher = None + + +a = Analysis(['src\\start_app.py'], + pathex=[], + binaries=[], + datas=[], + hiddenimports=[], + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False) +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) + +exe = EXE(pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + [], + name='MOP', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True, + disable_windowed_traceback=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None ) diff --git a/tasks.py b/tasks.py new file mode 100644 index 0000000..85062a6 --- /dev/null +++ b/tasks.py @@ -0,0 +1,73 @@ +""" +""" + +# --------------------------------------- # +# imports # +# --------------------------------------- # +from invoke import task + +import os +import shutil + +# --------------------------------------- # +# definitions # +# --------------------------------------- # +VIRTUALENV_NAME = "py39_MailOrderParser" +EXECUTABLE_NAME = "MOP" + +# --------------------------------------- # +# global vars # +# --------------------------------------- # + + +# --------------------------------------- # +# functions # +# --------------------------------------- # +def remove_temporary_folders(): + print("-> remove unused folders") + for folder in ["dist", "build", "temp"]: + if os.path.exists(folder): + print("remove: {}".format(folder)) + shutil.rmtree(folder) + print("finished!") + + +# --------------------------------------- # +# classes # +# --------------------------------------- # +@task +def update_requirements(cmd): + with cmd.prefix("workon {}".format(VIRTUALENV_NAME)): + cmd.run("pip freeze > requirements.txt") + + +@task +def create_exe(c, version="v9-9-9"): + with c.prefix("workon {}".format(VIRTUALENV_NAME)): + + print("---------- START CREATING EXE ----------") + remove_temporary_folders() + + print("-> start creating .exe") + c.run("pyinstaller start_app.spec") + print("finished!") + + print("-> start creating temporary folders and copy files") + for folder in ["temp", "temp/apps"]: + os.mkdir(folder) + + shutil.copyfile("dist/{}.exe".format(EXECUTABLE_NAME), "temp/apps/{}.exe".format(EXECUTABLE_NAME)) + + print("finished!") + print("-> start creating .zip") + + zip_name = EXECUTABLE_NAME + "_" + version + + shutil.make_archive(zip_name, "zip", "temp") + print("finished!") + remove_temporary_folders() + print("---------- FINISHED CREATING EXE ----------") + +# --------------------------------------- # +# main # +# --------------------------------------- # diff --git a/test/data/test_mail.txt b/test/data/test_mail.txt new file mode 100644 index 0000000..922d9f4 --- /dev/null +++ b/test/data/test_mail.txt @@ -0,0 +1,56 @@ + + + + +Von meinem/meiner Galaxy gesendet + + + +-------- Ursprüngliche Nachricht -------- +Von: Stefan Erne +Datum: 05.01.22 11:24 (GMT+01:00) +An: Dressel Bernhard +Betreff: Fwd: Nachricht über https://haus-gruenerwald-net.jimdofree.com/bio-bauernhof/ + + +Von meinem iPhone gesendet + +Anfang der weitergeleiteten Nachricht: +Von: haus-gruenerwald@gmx.net +Datum: 5. Jänner 2022 um 11:22:21 MEZ +An: stefan_erne@gmx.net +Betreff: WG: Nachricht über https://haus-gruenerwald-net.jimdofree.com/bio-bauernhof/ + + +-----Ursprüngliche Nachricht----- +Von: Jimdo +Gesendet: Mittwoch, 5. Januar 2022 11:18 +An: haus-gruenerwald@gmx.net +Betreff: Nachricht über https://haus-gruenerwald-net.jimdofree.com/bio-bauernhof/ + +Hallo, du hast eine Nachricht über deine Jimdo-Seite https://haus-gruenerwald-net.jimdofree.com/bio-bauernhof/ erhalten: + +------------------------------------- + +Rasse: BIO-Angus + +Paketgröße: 5 kg + +Vakuumiert: ja + +Option: Suppenfleisch + +Verpackungsgröße pro Beutel: 500 g + +Sonstiges (nach Verfügbarkeit): Backen (EUR 20,-/kg), Knochen (gratis) + +Name: Bernhard Dresseo + +Telefonnummer: +436644073617 + +eMail-Adresse: bernhard.dressel@gmail.com + +Nachricht: + +Nutzer hat die Datenschutzerklärung akzeptiert. Datum/Uhrzeit: 2022-01-05 11:17:44 CET + diff --git a/test/test_mail_parser.py b/test/test_mail_parser.py new file mode 100644 index 0000000..cb0445b --- /dev/null +++ b/test/test_mail_parser.py @@ -0,0 +1,41 @@ +""" +""" + +# --------------------------------------- # +# imports # +# --------------------------------------- # +from unittest import TestCase + +from src.mail_parser import parse_mail_and_return_order + +# --------------------------------------- # +# definitions # +# --------------------------------------- # + + +# --------------------------------------- # +# global vars # +# --------------------------------------- # + + +# --------------------------------------- # +# functions # +# --------------------------------------- # + + +# --------------------------------------- # +# classes # +# --------------------------------------- # + + +# --------------------------------------- # +# main # +# --------------------------------------- # + + +class TestMailParser(TestCase): + def test_parse_mail_and_return_order(self): + with open("data/test_mail.txt", "r", encoding="utf-8") as test_mail: + test_order_data = parse_mail_and_return_order(test_mail.readlines()) + + self.assertEqual("Bernhard Dresseo", test_order_data["Name"])