From 9148e2068f7f27f60bb7c190a2b8c24a03d77715 Mon Sep 17 00:00:00 2001 From: CEF Server Date: Wed, 15 May 2024 00:01:58 +0000 Subject: [PATCH] add migrations and switch to sqlalchemy --- .gitignore | 2 + alembic.ini.example | 114 ++++++++++++++++++ alembic/README | 1 + alembic/env.py | 78 ++++++++++++ alembic/script.py.mako | 26 ++++ .../4677228ad413_create_users_table.py | 32 +++++ .../e6b8e42fa629_create_uploads_table.py | 30 +++++ cef_3M/__init__.py | 14 ++- cef_3M/sql.py | 58 ++------- cef_3M/sql_generated.py | 25 ++++ config.example.py | 5 - requirements.txt | 36 +++++- 12 files changed, 358 insertions(+), 63 deletions(-) create mode 100644 .gitignore create mode 100644 alembic.ini.example create mode 100644 alembic/README create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/4677228ad413_create_users_table.py create mode 100644 alembic/versions/e6b8e42fa629_create_uploads_table.py create mode 100644 cef_3M/sql_generated.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bb54b34 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +config.py +alembic.ini diff --git a/alembic.ini.example b/alembic.ini.example new file mode 100644 index 0000000..00aa699 --- /dev/null +++ b/alembic.ini.example @@ -0,0 +1,114 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python>=3.9 or backports.zoneinfo library. +# Any required deps can installed by adding `alembic[tz]` to the pip requirements +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = mysql+mysqldb://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/README b/alembic/README new file mode 100644 index 0000000..e0d0858 --- /dev/null +++ b/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration with an async dbapi. \ No newline at end of file diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..36112a3 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,78 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..fbc4b07 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/4677228ad413_create_users_table.py b/alembic/versions/4677228ad413_create_users_table.py new file mode 100644 index 0000000..7b85dc7 --- /dev/null +++ b/alembic/versions/4677228ad413_create_users_table.py @@ -0,0 +1,32 @@ +"""create users table + +Revision ID: 4677228ad413 +Revises: +Create Date: 2024-05-14 22:09:02.958787 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '4677228ad413' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "users", + sa.Column("username", sa.VARCHAR(64), primary_key=True), + sa.Column("password", sa.VARCHAR(128), nullable=False), + sa.Column("created_at", sa.TIMESTAMP(), server_default=sa.func.now()), + sa.Column("temporary", sa.BOOLEAN(), server_default=sa.sql.expression.true()) + ) + + +def downgrade() -> None: + op.drop_table("users") diff --git a/alembic/versions/e6b8e42fa629_create_uploads_table.py b/alembic/versions/e6b8e42fa629_create_uploads_table.py new file mode 100644 index 0000000..dc47c65 --- /dev/null +++ b/alembic/versions/e6b8e42fa629_create_uploads_table.py @@ -0,0 +1,30 @@ +"""create uploads table + +Revision ID: e6b8e42fa629 +Revises: 4677228ad413 +Create Date: 2024-05-14 22:19:04.824346 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'e6b8e42fa629' +down_revision: Union[str, None] = '4677228ad413' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "uploads", + sa.Column("hash", sa.CHAR(64), primary_key=True), + sa.Column("expiry", sa.TIMESTAMP(), server_default=sa.text("(current_timestamp() + interval 1 week)")), + ) + + +def downgrade() -> None: + op.drop_table("uploads") diff --git a/cef_3M/__init__.py b/cef_3M/__init__.py index 9640f53..41e6100 100644 --- a/cef_3M/__init__.py +++ b/cef_3M/__init__.py @@ -4,17 +4,18 @@ from minio import Minio import mimetypes import re -from . import sql from .auth import JWTBearer +from .sql import SessionMaker, Uploads from . import util import config +from datetime import datetime, timedelta minioClient = Minio( config.MINIO_ADDR, access_key=config.MINIO_ACCESS_KEY, secret_key=config.MINIO_SECRET_KEY, -).g +) app = FastAPI() app.add_middleware( @@ -34,12 +35,15 @@ async def upload(file: UploadFile, request: Request): if len(spl) == 2: safeFilename += "." + util.safeName.sub("_", spl[1]) sha = await util.SHA256(file) - if sql.SqlExecuteFetchOne("SELECT * FROM `uploads` WHERE `hash` = %s", sha): - sql.SqlExecute("UPDATE `uploads` SET `expiry` = (NOW() + INTERVAL 1 WEEK) WHERE `hash` = %s", sha) + session = SessionMaker() + if existing := session.query(Uploads).where(Uploads.hash == sha).first(): + existing.expiry = datetime.now() + timedelta(days=7) else: mime = mimetypes.guess_type(safeFilename) minioClient.put_object("uploads", sha, file.file, file.size, content_type=mime[0]) - sql.SqlExecute("INSERT INTO `uploads`(`hash`) VALUES (%s)", sha) + up = Uploads(hash=sha) + session.add(up) + session.commit() return {"url": f"https://{config.MINIO_ADDR}/uploads/{sha}/{safeFilename}"} diff --git a/cef_3M/sql.py b/cef_3M/sql.py index 657eee9..9f2a676 100644 --- a/cef_3M/sql.py +++ b/cef_3M/sql.py @@ -1,52 +1,14 @@ -import pymysql -import config -from typing import Tuple +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from .sql_generated import * +import configparser -pymysql.install_as_MySQLdb() +alembic = configparser.ConfigParser() +alembic.read("alembic.ini") -import MySQLdb as maraidb +engine = create_engine( + alembic.get("alembic", "sqlalchemy.url") +) -DB: pymysql = maraidb.connect(user=config.MARIADB_USER, password=config.MARIADB_PASSWORD, db=config.MARIADB_DB, autocommit=True) -DB.autocommit(True) +SessionMaker = sessionmaker(autocommit=False, autoflush=False, bind=engine) -def reconnect(f): - def wrap(*args, **kwargs): - DB.ping() - return f(*args, **kwargs) - return wrap - -@reconnect -def SqlExecute(query, *args): - cursor = DB.cursor(pymysql.cursors.DictCursor) - cursor.execute(query, args) - cursor.close() - return cursor.lastrowid - -@reconnect -def SqlExecuteFetchOne(query, *args): - cursor = DB.cursor(pymysql.cursors.DictCursor) - cursor.execute(query, args) - row = cursor.fetchone() - cursor.close() - return row - -@reconnect -def MultipleSqlExecuteFetchOne(*queries: Tuple[str, tuple]): - cursor = DB.cursor(pymysql.cursors.DictCursor) - ret = [] - for query, args in queries: - cursor.execute(query, args) - ret.append(cursor.fetchone()) - cursor.close() - return ret - -@reconnect -def SqlExecuteFetchAll(query, *args): - cursor = DB.cursor(pymysql.cursors.DictCursor) - cursor.execute(query, args) - rows = cursor.fetchall() - cursor.close() - return rows - - -CACHE = {} \ No newline at end of file diff --git a/cef_3M/sql_generated.py b/cef_3M/sql_generated.py new file mode 100644 index 0000000..9a80366 --- /dev/null +++ b/cef_3M/sql_generated.py @@ -0,0 +1,25 @@ +from typing import Optional + +from sqlalchemy import CHAR, String, TIMESTAMP, text +from sqlalchemy.dialects.mysql import TINYINT +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column +import datetime + +class Base(DeclarativeBase): + pass + + +class Uploads(Base): + __tablename__ = 'uploads' + + hash: Mapped[str] = mapped_column(CHAR(64), primary_key=True) + expiry: Mapped[Optional[datetime.datetime]] = mapped_column(TIMESTAMP, server_default=text('(current_timestamp()')) + + +class Users(Base): + __tablename__ = 'users' + + username: Mapped[str] = mapped_column(String(64), primary_key=True) + password: Mapped[str] = mapped_column(String(128)) + created_at: Mapped[Optional[datetime.datetime]] = mapped_column(TIMESTAMP, server_default=text('current_timestamp()')) + temporary: Mapped[Optional[int]] = mapped_column(TINYINT(1), server_default=text('1')) diff --git a/config.example.py b/config.example.py index a9ff421..412ce1c 100644 --- a/config.example.py +++ b/config.example.py @@ -5,11 +5,6 @@ MINIO_ADDR = "data.example.xyz" MINIO_ACCESS_KEY = "access-key-goes-here" MINIO_SECRET_KEY = "secret-key-goes-here" -MARIADB_URL = "localhost" -MARIADB_USER = "ergo" -MARIADB_DB = "ergo_ext" -MARIADB_PASSWORD = "password-goes-here" - MAX_FILE_SIZE = 1024*1024*20 # Need to figure out how to make this cooperate more ALLOWED_DOMAINS = ["*"] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 7258bd7..123d3e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,47 @@ +alembic==1.13.1 annotated-types==0.6.0 anyio==3.7.1 certifi==2023.7.22 cffi==1.16.0 click==8.1.7 -cryptography==41.0.4 -fastapi==0.103.2 +cryptography==42.0.7 +dnspython==2.6.1 +email_validator==2.1.1 +fastapi==0.111.0 +fastapi-cli==0.0.3 +greenlet==3.0.3 h11==0.14.0 +httpcore==1.0.5 +httptools==0.6.1 +httpx==0.27.0 idna==3.4 +Jinja2==3.1.4 +Mako==1.3.5 +markdown-it-py==3.0.0 +MarkupSafe==2.1.5 +mdurl==0.1.2 minio==7.1.17 +mysqlclient==2.2.4 +orjson==3.10.3 pycparser==2.21 pydantic==2.4.2 pydantic_core==2.10.1 +Pygments==2.18.0 PyJWT==2.8.0 PyMySQL==1.1.0 -python-multipart==0.0.6 +python-dotenv==1.0.1 +python-multipart==0.0.9 +PyYAML==6.0.1 +rich==13.7.1 +shellingham==1.5.4 sniffio==1.3.0 -starlette==0.27.0 +SQLAlchemy==2.0.30 +starlette==0.37.2 +typer==0.12.3 typing_extensions==4.8.0 -urllib3==2.0.6 +ujson==5.10.0 +urllib3==2.2.1 uvicorn==0.23.2 +uvloop==0.19.0 +watchfiles==0.21.0 +websockets==12.0