Compare commits

...

14 Commits
main ... ha

6 changed files with 191 additions and 30 deletions

View File

@ -2,7 +2,7 @@ Package: fastapi-dls
Version: 0.0
Architecture: all
Maintainer: Oscar Krause oscar.krause@collinwebdesigns.de
Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-jose, python3-sqlalchemy, python3-pycryptodome, python3-markdown, uvicorn, openssl
Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-jose, python3-sqlalchemy, python3-pycryptodome, python3-markdown, python3-httpx, uvicorn, openssl
Recommends: curl
Installed-Size: 10240
Homepage: https://git.collinwebdesigns.de/oscar.krause/fastapi-dls

View File

@ -32,6 +32,17 @@ Tested with Ubuntu 22.10 (from Proxmox templates), actually its consuming 100mb
- Make sure your timezone is set correct on you fastapi-dls server and your client
**HA Setup Notes**
- only *failover mode* is supported by team-green (see *high availability* in official user guide)
- make sure you're using same configuration on each node
- use same `instance.private.pem` and `instance.private.key` on each node
- add `cronjob` on each node with `curl -X GET --insecure https://localhost/-/ha/replicate`
If you want to use *real* HA, you should use a proxy in front of this service and use a clustered database in backend.
This is not documented and supported by me, but it *can* work. Please ask the community for help.
Maybe the simplest solution for HA-ing this service is to use a Docker-Swarm with redundant storage and database.
## Docker
Docker-Images are available here:
@ -371,22 +382,27 @@ After first success you have to replace `--issue` with `--renew`.
# Configuration
| Variable | Default | Usage |
|------------------------|----------------------------------------|------------------------------------------------------------------------------------------------------|
|------------------------|----------------------------------------|--------------------------------------------------------------------------------------------------------------------|
| `DEBUG` | `false` | Toggles `fastapi` debug mode |
| `DLS_URL` | `localhost` | Used in client-token to tell guest driver where dls instance is reachable |
| `DLS_PORT` | `443` | Used in client-token to tell guest driver where dls instance is reachable |
| `HA_REPLICATE` | | `DLS_URL` + `DLS_PORT` of primary DLS instance, e.g. `dls-node:443` (for HA only **two** nodes are supported!) \*1 |
| `HA_ROLE` | | `PRIMARY` or `SECONDARY` |
| `TOKEN_EXPIRE_DAYS` | `1` | Client auth-token validity (used for authenticate client against api, **not `.tok` file!**) |
| `LEASE_EXPIRE_DAYS` | `90` | Lease time in days |
| `LEASE_RENEWAL_PERIOD` | `0.15` | The percentage of the lease period that must elapse before a licensed client can renew a license \*1 |
| `LEASE_RENEWAL_PERIOD` | `0.15` | The percentage of the lease period that must elapse before a licensed client can renew a license \*2 |
| `DATABASE` | `sqlite:///db.sqlite` | See [official SQLAlchemy docs](https://docs.sqlalchemy.org/en/14/core/engines.html) |
| `CORS_ORIGINS` | `https://{DLS_URL}` | Sets `Access-Control-Allow-Origin` header (comma separated string) \*2 |
| `CORS_ORIGINS` | `https://{DLS_URL}` | Sets `Access-Control-Allow-Origin` header (comma separated string) \*3 |
| `SITE_KEY_XID` | `00000000-0000-0000-0000-000000000000` | Site identification uuid |
| `INSTANCE_REF` | `10000000-0000-0000-0000-000000000001` | Instance identification uuid |
| `ALLOTMENT_REF` | `20000000-0000-0000-0000-000000000001` | Allotment identification uuid |
| `INSTANCE_KEY_RSA` | `<app-dir>/cert/instance.private.pem` | Site-wide private RSA key for singing JWTs \*3 |
| `INSTANCE_KEY_PUB` | `<app-dir>/cert/instance.public.pem` | Site-wide public key \*3 |
| `INSTANCE_KEY_RSA` | `<app-dir>/cert/instance.private.pem` | Site-wide private RSA key for singing JWTs \*4 |
| `INSTANCE_KEY_PUB` | `<app-dir>/cert/instance.public.pem` | Site-wide public key \*4 |
\*1 For example, if the lease period is one day and the renewal period is 20%, the client attempts to renew its license
\*1 If you want to use HA, this value should be point to `secondary` on `primary` and `primary` on `secondary`. Don't
use same database for both instances!
\*2 For example, if the lease period is one day and the renewal period is 20%, the client attempts to renew its license
every 4.8 hours. If network connectivity is lost, the loss of connectivity is detected during license renewal and the
client has 19.2 hours in which to re-establish connectivity before its license expires.

View File

@ -6,7 +6,7 @@ from os.path import join, dirname
from os import getenv as env
from dotenv import load_dotenv
from fastapi import FastAPI
from fastapi import FastAPI, BackgroundTasks
from fastapi.requests import Request
from json import loads as json_loads
from datetime import datetime, timedelta
@ -19,7 +19,7 @@ from starlette.responses import StreamingResponse, JSONResponse as JSONr, HTMLRe
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from util import load_key, load_file
from util import load_key, load_file, ha_replicate
from orm import Origin, Lease, init as db_init, migrate
load_dotenv('../version.env')
@ -36,6 +36,7 @@ db_init(db), migrate(db)
# everything prefixed with "INSTANCE_*" is used as "SERVICE_INSTANCE_*" or "SI_*" in official dls service
DLS_URL = str(env('DLS_URL', 'localhost'))
DLS_PORT = int(env('DLS_PORT', '443'))
HA_REPLICATE, HA_ROLE = env('HA_REPLICATE', None), env('HA_ROLE', None) # only failover is supported
SITE_KEY_XID = str(env('SITE_KEY_XID', '00000000-0000-0000-0000-000000000000'))
INSTANCE_REF = str(env('INSTANCE_REF', '10000000-0000-0000-0000-000000000001'))
ALLOTMENT_REF = str(env('ALLOTMENT_REF', '20000000-0000-0000-0000-000000000001'))
@ -199,6 +200,36 @@ async def _client_token():
cur_time = datetime.utcnow()
exp_time = cur_time + CLIENT_TOKEN_EXPIRE_DELTA
if HA_REPLICATE is not None and HA_ROLE.lower() == "secondary":
return RedirectResponse(f'https://{HA_REPLICATE}/-/client-token')
idx_port, idx_node = 0, 0
def create_svc_port_set(port: int):
idx = idx_port
return {
"idx": idx,
"d_name": "DLS",
"svc_port_map": [{"service": "auth", "port": port}, {"service": "lease", "port": port}]
}
def create_node_url(url: str, svc_port_set_idx: int):
idx = idx_node
return {"idx": idx, "url": url, "url_qr": url, "svc_port_set_idx": svc_port_set_idx}
service_instance_configuration = {
"nls_service_instance_ref": INSTANCE_REF,
"svc_port_set_list": [create_svc_port_set(DLS_PORT)],
"node_url_list": [create_node_url(DLS_URL, idx_port)]
}
idx_port += 1
idx_node += 1
if HA_REPLICATE is not None and HA_ROLE.lower() == "primary":
SEC_URL, SEC_PORT, *invalid = HA_REPLICATE.split(':')
service_instance_configuration['svc_port_set_list'].append(create_svc_port_set(SEC_PORT))
service_instance_configuration['node_url_list'].append(create_node_url(SEC_URL, idx_port))
payload = {
"jti": str(uuid4()),
"iss": "NLS Service Instance",
@ -209,17 +240,7 @@ async def _client_token():
"update_mode": "ABSOLUTE",
"scope_ref_list": [ALLOTMENT_REF],
"fulfillment_class_ref_list": [],
"service_instance_configuration": {
"nls_service_instance_ref": INSTANCE_REF,
"svc_port_set_list": [
{
"idx": 0,
"d_name": "DLS",
"svc_port_map": [{"service": "auth", "port": DLS_PORT}, {"service": "lease", "port": DLS_PORT}]
}
],
"node_url_list": [{"idx": 0, "url": DLS_URL, "url_qr": DLS_URL, "svc_port_set_idx": 0}]
},
"service_instance_configuration": service_instance_configuration,
"service_instance_public_key_configuration": {
"service_instance_public_key_me": {
"mod": hex(INSTANCE_KEY_PUB.public_key().n)[2:],
@ -239,6 +260,67 @@ async def _client_token():
return response
@app.get('/-/ha/replicate', summary='* HA replicate - trigger')
async def _ha_replicate_to_ha(request: Request, background_tasks: BackgroundTasks):
if HA_REPLICATE is None or HA_ROLE is None:
logger.warning('HA replicate endpoint triggerd, but no value for "HA_REPLICATE" or "HA_ROLE" is set!')
return JSONr(status_code=503, content={'status': 503, 'detail': 'no value for "HA_REPLICATE" or "HA_ROLE" set'})
session = sessionmaker(bind=db)()
origins = [origin.serialize() for origin in session.query(Origin).all()]
leases = [lease.serialize(renewal_period=LEASE_RENEWAL_PERIOD, renewal_delta=LEASE_RENEWAL_DELTA) for lease in session.query(Lease).all()]
background_tasks.add_task(ha_replicate, logger, HA_REPLICATE, HA_ROLE, VERSION, DLS_URL, DLS_PORT, SITE_KEY_XID, INSTANCE_REF, origins, leases)
return JSONr(status_code=202, content=None)
@app.put('/-/ha/replicate', summary='* HA replicate')
async def _ha_replicate_by_ha(request: Request):
j, cur_time = json_loads((await request.body()).decode('utf-8')), datetime.utcnow()
if HA_REPLICATE is None:
logger.warning(f'HA replicate endpoint triggerd, but no value for "HA_REPLICATE" is set!')
return JSONr(status_code=503, content={'status': 503, 'detail': 'no value for "HA_REPLICATE" set'})
version = j.get('VERSION')
if version != VERSION:
logger.error(f'Version missmatch on HA replication task!')
return JSONr(status_code=503, content={'status': 503, 'detail': 'Missmatch for "VERSION"'})
site_key_xid = j.get('SITE_KEY_XID')
if site_key_xid != SITE_KEY_XID:
logger.error(f'Site-Key missmatch on HA replication task!')
return JSONr(status_code=503, content={'status': 503, 'detail': 'Missmatch for "SITE_KEY_XID"'})
instance_ref = j.get('INSTANCE_REF')
if instance_ref != INSTANCE_REF:
logger.error(f'Version missmatch on HA replication task!')
return JSONr(status_code=503, content={'status': 503, 'detail': 'Missmatch for "INSTANCE_REF"'})
sync_timestamp, max_seconds_behind = datetime.fromisoformat(j.get('sync_timestamp')), 30
if sync_timestamp <= cur_time - timedelta(seconds=max_seconds_behind):
logger.error(f'Request time more than {max_seconds_behind}s behind!')
return JSONr(status_code=503, content={'status': 503, 'detail': 'Request time behind'})
origins, leases = j.get('origins'), j.get('leases')
for origin in origins:
origin_ref = origin.get('origin_ref')
logging.info(f'> [ ha ]: origin {origin_ref}')
data = Origin.deserialize(origin)
Origin.create_or_update(db, data)
for lease in leases:
lease_ref = lease.get('lease_ref')
x = Lease.find_by_lease_ref(db, lease_ref)
if x is not None and x.lease_updated > sync_timestamp:
continue
logging.info(f'> [ ha ]: lease {lease_ref}')
data = Lease.deserialize(lease)
Lease.create_or_update(db, data)
return JSONr(status_code=202, content=None)
# venv/lib/python3.9/site-packages/nls_services_auth/test/test_origins_controller.py
@app.post('/auth/v1/origin', description='find or create an origin')
async def auth_v1_origin(request: Request):
@ -545,6 +627,22 @@ async def app_on_startup():
Your client-token file (.tok) is valid for {str(CLIENT_TOKEN_EXPIRE_DELTA)}.
''')
if HA_REPLICATE is not None and HA_ROLE is not None:
from hashlib import sha1
sha1digest = sha1(INSTANCE_KEY_RSA.export_key()).hexdigest()
fingerprint_key = ':'.join(sha1digest[i: i + 2] for i in range(0, len(sha1digest), 2))
sha1digest = sha1(INSTANCE_KEY_PUB.export_key()).hexdigest()
fingerprint_pub = ':'.join(sha1digest[i: i + 2] for i in range(0, len(sha1digest), 2))
logger.info(f'''
HA mode is enabled. Make sure theses fingerprints matches on all your nodes:
- INSTANCE_KEY_RSA: "{str(fingerprint_key)}"
- INSTANCE_KEY_PUB: "{str(fingerprint_pub)}"
This node ({HA_ROLE}) listens to "https://{DLS_URL}:{DLS_PORT}" and replicates to "https://{HA_REPLICATE}".
''')
if __name__ == '__main__':
import uvicorn

View File

@ -32,6 +32,16 @@ class Origin(Base):
'os_version': self.os_version,
}
@staticmethod
def deserialize(j) -> "Origin":
return Origin(
origin_ref=j.get('origin_ref'),
hostname=j.get('hostname'),
guest_driver_version=j.get('guest_driver_version'),
os_platform=j.get('os_platform'),
os_version=j.get('os_version'),
)
@staticmethod
def create_statement(engine: Engine):
from sqlalchemy.schema import CreateTable
@ -95,6 +105,16 @@ class Lease(Base):
'lease_renewal': lease_renewal.isoformat(),
}
@staticmethod
def deserialize(j) -> "Lease":
return Lease(
lease_ref=j.get('lease_ref'),
origin_ref=j.get('origin_ref'),
lease_created=datetime.fromisoformat(j.get('lease_created')),
lease_expires=datetime.fromisoformat(j.get('lease_expires')),
lease_updated=datetime.fromisoformat(j.get('lease_updated')),
)
@staticmethod
def create_statement(engine: Engine):
from sqlalchemy.schema import CreateTable

View File

@ -26,3 +26,29 @@ def generate_key() -> "RsaKey":
from Cryptodome.PublicKey.RSA import RsaKey
return RSA.generate(bits=2048)
def ha_replicate(logger: "logging.Logger", ha_replicate: str, ha_role: str, version: str, dls_url: str, dls_port: int, site_key_xid: str, instance_ref: str, origins: list, leases: list) -> bool:
from datetime import datetime
import httpx
if f'{dls_url}:{dls_port}' == ha_replicate:
logger.error(f'Failed to replicate this node ({ha_role}) to "{ha_replicate}": can\'t replicate to itself')
return False
data = {
'VERSION': str(version),
'HA_REPLICATE': f'{dls_url}:{dls_port}',
'SITE_KEY_XID': str(site_key_xid),
'INSTANCE_REF': str(instance_ref),
'origins': origins,
'leases': leases,
'sync_timestamp': datetime.utcnow().isoformat(),
}
r = httpx.put(f'https://{ha_replicate}/-/ha/replicate', json=data, verify=False)
if r.status_code == 202:
logger.info(f'Successfully replicated this node ({ha_role}) to "{ha_replicate}".')
return True
logger.error(f'Failed to replicate this node ({ha_role}) to "{ha_replicate}": {r.status_code} - {r.content}')
return False

View File

@ -6,3 +6,4 @@ python-dateutil==2.8.2
sqlalchemy==2.0.3
markdown==3.4.1
python-dotenv==0.21.1
httpx==0.23.3