From 9c4114498583afab1f133936135f8bc7a632d9cd Mon Sep 17 00:00:00 2001 From: CEF Server Date: Sat, 11 May 2024 07:40:39 +0000 Subject: [PATCH] voice state - also there was no git history here so im just force pushing --- cef_3M/__init__.py | 46 ++++++++++++++++++++++++++++++++++++++++ cef_3M/auth.py | 45 +++++++++++++++++++++++++++++++++++++++ cef_3M/sql.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++ cef_3M/util.py | 15 +++++++++++++ config.example.py | 15 +++++++++++++ main.py | 3 +++ requirements.txt | 21 +++++++++++++++++++ run.sh | 2 ++ scripts/cleanup.py | 8 +++++++ 9 files changed, 207 insertions(+) create mode 100644 cef_3M/__init__.py create mode 100644 cef_3M/auth.py create mode 100644 cef_3M/sql.py create mode 100644 cef_3M/util.py create mode 100644 config.example.py create mode 100644 main.py create mode 100644 requirements.txt create mode 100755 run.sh create mode 100644 scripts/cleanup.py diff --git a/cef_3M/__init__.py b/cef_3M/__init__.py new file mode 100644 index 0000000..9640f53 --- /dev/null +++ b/cef_3M/__init__.py @@ -0,0 +1,46 @@ +from fastapi import FastAPI, UploadFile, Request, Depends +from fastapi.middleware.cors import CORSMiddleware +from minio import Minio +import mimetypes +import re + +from . import sql +from .auth import JWTBearer +from . import util +import config + + +minioClient = Minio( + config.MINIO_ADDR, + access_key=config.MINIO_ACCESS_KEY, + secret_key=config.MINIO_SECRET_KEY, +).g + +app = FastAPI() +app.add_middleware( + CORSMiddleware, + allow_origins=config.ALLOWED_DOMAINS, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +@app.post("/upload", dependencies=[Depends(JWTBearer())]) +async def upload(file: UploadFile, request: Request): + if file.size > config.MAX_FILE_SIZE: + return {"error": "file too big"} + spl = file.filename.rsplit(".", 1) + safeFilename = util.safeName.sub("_", spl[0]) + if len(spl) == 2: + safeFilename += "." + util.safeName.sub("_", spl[1]) + sha = await util.SHA256(file) + if sql.SqlExecuteFetchOne("SELECT * FROM `uploads` WHERE `hash` = %s", sha): + sql.SqlExecute("UPDATE `uploads` SET `expiry` = (NOW() + INTERVAL 1 WEEK) WHERE `hash` = %s", sha) + else: + mime = mimetypes.guess_type(safeFilename) + minioClient.put_object("uploads", sha, file.file, file.size, content_type=mime[0]) + sql.SqlExecute("INSERT INTO `uploads`(`hash`) VALUES (%s)", sha) + return {"url": f"https://{config.MINIO_ADDR}/uploads/{sha}/{safeFilename}"} + + +__all__ = ["sql", "auth", "util"] \ No newline at end of file diff --git a/cef_3M/auth.py b/cef_3M/auth.py new file mode 100644 index 0000000..2c66e93 --- /dev/null +++ b/cef_3M/auth.py @@ -0,0 +1,45 @@ +import time +import jwt +from fastapi.security import HTTPBearer + +import config +from fastapi import Request, HTTPException + +JWT_PUBKEY = open(config.SECRETKEY).read() +JWT_ALGORITHM = "RS256" + + +def decodeJWT(token: str) -> dict: + try: + decoded_token = jwt.decode(token, JWT_PUBKEY, algorithms=[JWT_ALGORITHM]) + return decoded_token if decoded_token["exp"] >= time.time() else None + except: + return {} + + +class JWTBearer(HTTPBearer): + def __init__(self, auto_error: bool = True): + super(JWTBearer, self).__init__(auto_error=auto_error) + + async def __call__(self, request: Request): + credentials = await super(JWTBearer, self).__call__(request) + if credentials: + if not credentials.scheme == "Bearer": + raise HTTPException(status_code=403, detail="Invalid authentication scheme.") + if not self.verify_jwt(credentials.credentials): + raise HTTPException(status_code=403, detail="Invalid or expired token.") + request.state.jwt = decodeJWT(credentials.credentials) + return credentials.credentials + else: + raise HTTPException(status_code=403, detail="Invalid authorization code.") + + def verify_jwt(self, jwtoken: str) -> bool: + isTokenValid: bool = False + + try: + payload = decodeJWT(jwtoken) + except: + payload = None + if payload: + isTokenValid = True + return isTokenValid diff --git a/cef_3M/sql.py b/cef_3M/sql.py new file mode 100644 index 0000000..657eee9 --- /dev/null +++ b/cef_3M/sql.py @@ -0,0 +1,52 @@ +import pymysql +import config +from typing import Tuple + +pymysql.install_as_MySQLdb() + +import MySQLdb as maraidb + +DB: pymysql = maraidb.connect(user=config.MARIADB_USER, password=config.MARIADB_PASSWORD, db=config.MARIADB_DB, autocommit=True) +DB.autocommit(True) + +def reconnect(f): + def wrap(*args, **kwargs): + DB.ping() + return f(*args, **kwargs) + return wrap + +@reconnect +def SqlExecute(query, *args): + cursor = DB.cursor(pymysql.cursors.DictCursor) + cursor.execute(query, args) + cursor.close() + return cursor.lastrowid + +@reconnect +def SqlExecuteFetchOne(query, *args): + cursor = DB.cursor(pymysql.cursors.DictCursor) + cursor.execute(query, args) + row = cursor.fetchone() + cursor.close() + return row + +@reconnect +def MultipleSqlExecuteFetchOne(*queries: Tuple[str, tuple]): + cursor = DB.cursor(pymysql.cursors.DictCursor) + ret = [] + for query, args in queries: + cursor.execute(query, args) + ret.append(cursor.fetchone()) + cursor.close() + return ret + +@reconnect +def SqlExecuteFetchAll(query, *args): + cursor = DB.cursor(pymysql.cursors.DictCursor) + cursor.execute(query, args) + rows = cursor.fetchall() + cursor.close() + return rows + + +CACHE = {} \ No newline at end of file diff --git a/cef_3M/util.py b/cef_3M/util.py new file mode 100644 index 0000000..f03ec2e --- /dev/null +++ b/cef_3M/util.py @@ -0,0 +1,15 @@ +import hashlib +import re + +from fastapi import UploadFile + + +safeName = re.compile(r"[^\w\d\.-]") + +# If this gets too out of hand, put an async breakpoint to allow other things to be handled while the hash occurs +async def SHA256(f: UploadFile) -> str: + sha = hashlib.sha256() + while data := await f.read(65535): + sha.update(data) + await f.seek(0) + return sha.hexdigest() diff --git a/config.example.py b/config.example.py new file mode 100644 index 0000000..a9ff421 --- /dev/null +++ b/config.example.py @@ -0,0 +1,15 @@ +import os +SECRETKEY = os.path.join("secrets", "pubkey.pem") + +MINIO_ADDR = "data.example.xyz" +MINIO_ACCESS_KEY = "access-key-goes-here" +MINIO_SECRET_KEY = "secret-key-goes-here" + +MARIADB_URL = "localhost" +MARIADB_USER = "ergo" +MARIADB_DB = "ergo_ext" +MARIADB_PASSWORD = "password-goes-here" + +MAX_FILE_SIZE = 1024*1024*20 +# Need to figure out how to make this cooperate more +ALLOWED_DOMAINS = ["*"] \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..e0f0c77 --- /dev/null +++ b/main.py @@ -0,0 +1,3 @@ +import uvicorn +import cef_3M +uvicorn.run("cef_3M:app", port=8001, reload=True) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7258bd7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,21 @@ +annotated-types==0.6.0 +anyio==3.7.1 +certifi==2023.7.22 +cffi==1.16.0 +click==8.1.7 +cryptography==41.0.4 +fastapi==0.103.2 +h11==0.14.0 +idna==3.4 +minio==7.1.17 +pycparser==2.21 +pydantic==2.4.2 +pydantic_core==2.10.1 +PyJWT==2.8.0 +PyMySQL==1.1.0 +python-multipart==0.0.6 +sniffio==1.3.0 +starlette==0.27.0 +typing_extensions==4.8.0 +urllib3==2.0.6 +uvicorn==0.23.2 diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..f9a97d4 --- /dev/null +++ b/run.sh @@ -0,0 +1,2 @@ +#!/bin/sh +uvicorn --reload main:app --port 8001 diff --git a/scripts/cleanup.py b/scripts/cleanup.py new file mode 100644 index 0000000..8f4fc79 --- /dev/null +++ b/scripts/cleanup.py @@ -0,0 +1,8 @@ +from cef_3M import sql, minioClient + +# This should be run every hour or so to clean up old uploads +toDelete = sql.SqlExecuteFetchAll("SELECT *, NOW() FROM uploads WHERE expiry < NOW()") +for f in toDelete: + minioClient.remove_object("uploads", f["hash"]) + sql.SqlExecute("DELETE FROM `uploads` WHERE `hash` = %s", f["hash"]) +print(f"Deleted {len(toDelete)} old files")