master
nixo 2020-12-10 14:49:04 +01:00
commit 21132d90b7
6 changed files with 2295 additions and 0 deletions

10
readme.org Normal file
View File

@ -0,0 +1,10 @@
* How to run
Create dirs for postgres (=/srv/pgdata-weblate=) and weblate
(=/var/lib/weblate=) data and run
#+begin_src shell
./setup.sh
#+end_src
which will start a container with all required services.

211
settings.py Normal file
View File

@ -0,0 +1,211 @@
#
# Copyright © 2012 - 2020 Michal Čihař <michal@cihar.com>
#
# This file is part of Weblate <https://weblate.org/>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
import os
import platform
from logging.handlers import SysLogHandler
from weblate.settings_example import * # noqa
ADMINS = (("Weblate Admin", "noreply@your-host.xyz"),)
MANAGERS = ADMINS
DATABASES = {
"default": {
# Use "postgresql" or "mysql".
"ENGINE": "django.db.backends.postgresql",
# Database name.
"NAME": "weblate",
# Database user.
"USER": "weblate",
# Name of role to alter to set parameters in PostgreSQL,
# use in case role name is different than user used for authentication.
# "ALTER_ROLE": "weblate",
# Database password.
# "PASSWORD": "",
# Set to empty string for localhost.
"HOST": "localhost",
# Set to empty string for default.
"PORT": "5431",
# Customizations for databases.
"OPTIONS": {
# In case of using an older MySQL server,
# which has MyISAM as a default storage
# "init_command": "SET storage_engine=INNODB",
# Uncomment for MySQL older than 5.7:
# "init_command": "SET sql_mode='STRICT_TRANS_TABLES'",
# Set emoji capable charset for MySQL:
# "charset": "utf8mb4",
# Change connection timeout in case you get MySQL gone away error:
# "connect_timeout": 28800,
},
}
}
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Data Directory
DATA_DIR = "/var/lib/weblate"
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = "UTC"
# ENABLE THIS 3 IN PRODUCTION AND USE NGINX WITH SSL ENABLED
ENABLE_HTTPS = False
SECURE_SSL_REDIRECT = False
SESSION_COOKIE_SECURE = False
SECRET_KEY = "RANDOM SECRET KEY"
SITE_DOMAIN = "your-host.xyz"
MEDIA_ROOT = os.path.join(DATA_DIR, "media")
STATIC_ROOT = os.path.join(DATA_DIR, "static")
CELERY_BEAT_SCHEDULE_FILENAME = os.path.join(DATA_DIR, "celery", "beat-schedule")
CELERY_TASK_ALWAYS_EAGER = False
CELERY_BROKER_URL = "redis://localhost:6379"
CELERY_RESULT_BACKEND = CELERY_BROKER_URL
# Needed for makemessages, otherwise it does not discover all available locales
# and the -a parameter does not work
LOCALE_PATHS = [os.path.join(os.path.dirname(__file__), "locale")]
DEBUG = False
# Silent logging setup
LOGGING = {
"version": 1,
"disable_existing_loggers": True,
"filters": {"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}},
"formatters": {"simple": {"format": "%(levelname)s %(message)s"}},
"handlers": {
"mail_admins": {
"level": "ERROR",
"filters": ["require_debug_false"],
"class": "django.utils.log.AdminEmailHandler",
},
"console": {
"level": "DEBUG",
"class": "logging.StreamHandler",
"formatter": "simple",
},
},
"loggers": {
"django.request": {
"handlers": ["mail_admins"],
"level": "ERROR",
"propagate": True,
},
"weblate": {"handlers": [], "level": "ERROR"},
"social": {"handlers": [], "level": "ERROR"},
},
}
# Reset caches
CACHES = {
"default":
{
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": "redis://127.0.0.1:6379/0",
# 'LOCATION': 'unix:///var/run/redis/redis.sock?db=0',
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
"PARSER_CLASS": "redis.connection.HiredisParser",
}
},
"avatar": {
"BACKEND": "django.core.cache.backends.filebased.FileBasedCache",
"LOCATION": os.path.join(DATA_DIR, "avatar-cache"),
"TIMEOUT": 604800,
"OPTIONS": {
"MAX_ENTRIES": 1000,
},
}
}
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [
os.path.join(BASE_DIR, 'templates'),
],
'OPTIONS': {
'context_processors': [
'django.contrib.auth.context_processors.auth',
'django.template.context_processors.debug',
'django.template.context_processors.i18n',
'django.template.context_processors.request',
'django.template.context_processors.csrf',
'django.contrib.messages.context_processors.messages',
'weblate.trans.context_processors.weblate_context',
],
'loaders': [
('django.template.loaders.cached.Loader', [
'django.template.loaders.filesystem.Loader',
'django.template.loaders.app_directories.Loader',
]),
],
},
},
]
SESSION_COOKIE_HTTPONLY = True
# Use database backed sessions for transaction consistency in tests
SESSION_ENGINE = "django.contrib.sessions.backends.db"
# Use weak password hasher in tests, there is no point in spending CPU time
# in hashing test passwords
PASSWORD_HASHERS = [
'django.contrib.auth.hashers.PBKDF2PasswordHasher',
'django.contrib.auth.hashers.PBKDF2SHA1PasswordHasher',
'django.contrib.auth.hashers.Argon2PasswordHasher',
'django.contrib.auth.hashers.BCryptSHA256PasswordHasher',
'django.contrib.auth.hashers.CryptPasswordHasher'
]
AUTHENTICATION_BACKENDS = (
"social_core.backends.email.EmailAuth",
"social_core.backends.github.GithubOAuth2",
"social_core.backends.gitlab.GitLabOAuth2",
"weblate.accounts.auth.WeblateUserBackend",
)
SOCIAL_AUTH_GITLAB_KEY = ""
SOCIAL_AUTH_GITLAB_SECRET = ""
SOCIAL_AUTH_GITLAB_SCOPE = ["read_user"]
SOCIAL_AUTH_GITHUB_KEY = ""
SOCIAL_AUTH_GITHUB_SECRET = ""
SOCIAL_AUTH_GITHUB_SCOPE = ["user:email"]
AUTH_VALIDATE_PERMS = True
SERVER_EMAIL = ""
DEFAULT_FROM_EMAIL = ""
EMAIL_USE_TLS = True
EMAIL_HOST = "smtp.gmail.com"
EMAIL_PORT = 587
EMAIL_HOST_PASSWORD = ""
EMAIL_HOST_USER = ""

4
setup.sh Executable file
View File

@ -0,0 +1,4 @@
set -e
CMD=$(guix system container system.scm -L . --share=/srv/pgdata-weblate --share=/var/lib/weblate --network)
echo "Running: $CMD"
sudo $CMD

86
system.scm Normal file
View File

@ -0,0 +1,86 @@
;; Base system
(use-modules (gnu packages base))
(use-modules (gnu bootloader))
(use-modules (gnu bootloader grub))
(use-modules (gnu system file-systems))
(use-modules (guix packages))
;; Helpers
(use-modules (gnu packages curl))
(use-modules (gnu packages wget))
(use-modules (gnu packages python))
(use-modules (gnu packages gtk))
(use-modules (gnu packages glib))
(use-modules (gnu packages version-control))
;; Services (ssh, dhcp)
(use-modules (gnu services ssh))
(use-modules (gnu services networking))
(use-modules (gnu) (gnu system nss))
(use-modules (weblate))
(use-modules (weblate-service))
(use-modules (gnu services web))
(use-modules (gnu packages web))
(use-modules (gnu packages databases))
(use-modules (gnu services databases))
(use-package-modules databases geo)
(use-modules (gnu packages php))
(use-modules (gnu packages certs))
(use-modules (guix utils))
(define weblate-user
(user-account
(name "weblate")
(group "users")
(supplementary-groups '("netdev"))
(home-directory "/var/lib/weblate")))
(define (weblate/settings.py database-user website-url)
(local-file "settings.py"))
(operating-system
(host-name "weblate")
(timezone "Europe/Rome")
(locale "en_US.utf8")
(locale-libcs (list glibc-2.28 (canonical-package glibc)))
(bootloader (bootloader-configuration (bootloader grub-bootloader)))
(file-systems %base-file-systems)
(users
(cons* weblate-user
%base-user-accounts))
(packages
(cons*
pango cairo glib gobject-introspection git
nss-certs
%base-packages))
(services
(cons*
(service dhcp-client-service-type)
;; Weblate
(service redis-service-type)
(service weblate-service-type
(weblate-configuration
(settings-file (local-file "settings.py"))
(user "weblate")
(group "nginx")
(listen '("0.0.0.0:8765"))
(uwsgi-listen "0.0.0.0:8181")
(root "/var/lib/weblate")))
(service postgresql-service-type ;postgre
(postgresql-configuration
(postgresql postgresql)
(port 5431)
(data-directory "/srv/pgdata-weblate")
(config-file
(postgresql-config-file
(hba-file
(plain-file "pg_hba.conf"
"
local all all trust
host all all 127.0.0.1/32 trust
host all all ::1/128 trust"))))))
%base-services))
;; Allow resolution of '.local' host names with mDNS.
(name-service-switch %mdns-host-lookup-nss))

414
weblate-service.scm Normal file
View File

@ -0,0 +1,414 @@
(define-module (weblate-service)
#:use-module (weblate)
#:use-module (guix gexp)
#:use-module (guix utils)
#:use-module (guix build python-build-system)
#:use-module (guix packages)
#:use-module (gnu services)
#:use-module (gnu services web)
#:use-module (gnu packages version-control)
#:use-module (gnu packages web)
#:use-module (gnu packages python)
#:use-module (gnu packages ssh)
#:use-module (gnu packages databases)
#:use-module (gnu services shepherd)
#:use-module (guix records)
#:use-module (ice-9 match)
#:use-module (gnu system shadow)
#:export (weblate-configuration
weblate-configuration?
weblate-service-type))
(define-record-type* <weblate-configuration>
weblate-configuration make-weblate-configuration
weblate-configuration?
(settings-file weblate-configuration-settings-file)
(user weblate-configuration-user)
(group weblate-configuration-group)
(listen weblate-configuration-listen)
(root weblate-configuration-root)
(uwsgi-listen weblate-configuration-uwsgi-listen))
;; (define %weblate-accounts
;; (list (user-group (name "weblate") (system? #t))
;; (user-account
;; (name "weblate")
;; (group "weblate")
;; (system? #t)
;; (comment "Weblate server user")
;; (home-directory "/var/empty")
;; (shell (file-append shadow "/sbin/nologin")))))
(define (weblate-accounts config)
(let ((user (weblate-configuration-user config))
(group (weblate-configuration-group config)))
(list (user-group (name group) (system? #t))
;; FIXME: change default group in uwsgi.ini
(user-group (name "weblate") (system? #t))
(user-account
(name user)
(group group)
(system? #t)
(comment "Weblate service user")
(home-directory "/var/empty")
;; (shell (file-append shadow "/sbin/nologin"))
))))
(define weblate-service-type
(service-type (name 'weblate)
(description "Run WEBLATE")
(extensions
(list
(service-extension nginx-service-type weblate-nginx-service)
(service-extension account-service-type
weblate-accounts)
(service-extension activation-service-type
weblate-activation)
(service-extension shepherd-root-service-type
weblate-shepherd-services)))))
(define (weblate-shepherd-services config)
(list (weblate-initial-database-setup-service config)
(uwsgi-weblate-service config)
(celery-weblate-service config)))
;; TODO:
;; - Failed to find git-http-backend, the git exporter will not work.
;; - /var/lib/weblate/cache/fonts/
;; - /var/lib/weblate/home/.gitconfig
;; https://develop.sentry.dev/self-hosted/
(define (weblate-initial-database-setup-service config)
(define start-gexp
#~(lambda ()
(let ((pid (primitive-fork))
(postgres (getpwnam "postgres")))
(if (eq? pid 0)
(dynamic-wind
(const #t)
(lambda ()
(setgid (passwd:gid postgres))
(setuid (passwd:uid postgres))
(primitive-exit
(if (and
(zero?
(system* #$(file-append postgresql "/bin/createuser")
"--superuser" "weblate"))
(zero?
(system* #$(file-append postgresql "/bin/createdb")
"-O" "weblate" "weblate")))
0
1)))
(lambda ()
(primitive-exit 1)))
(zero? (cdr (waitpid pid)))))
(let ((pid (primitive-fork))
(weblate (getpwnam "weblate")))
(if (eq? pid 0)
(dynamic-wind
(const #t)
(lambda ()
(setgid (passwd:gid weblate))
(setuid (passwd:uid weblate))
(primitive-exit
(begin
(setenv "PYTHONPATH" #$(weblate-configuration-root config))
(setenv "DJANGO_SETTINGS_MODULE" "guix.settings")
(setenv "PATH" (string-append #$(file-append git) "/bin/"))
(if (and
(zero?
(system* #$(file-append python-weblate "/bin/weblate")
"migrate" "--no-input"))
(zero?
(system* #$(file-append python-weblate "/bin/weblate")
"createadmin" "--password" "default-password"
;; graceful manage existing admin
;; "--update"
)))
0
1))))
(lambda ()
(primitive-exit 1)))
(zero? (cdr (waitpid pid)))))))
(shepherd-service
(requirement '(postgres))
(provision '(weblate-initial-database-setup))
(start start-gexp)
(stop #~(const #f))
(respawn? #f)
(one-shot? #t)
(documentation "Setup Weblate database.")))
(define (weblate-activation config)
;; Activation gexp.
#~(begin
(use-modules (guix build utils))
(let* ((user #$(weblate-configuration-user config))
(group #$(weblate-configuration-group config))
(root #$(weblate-configuration-root config))
(settings-file #$(weblate-configuration-settings-file config)))
(when (not (file-exists? root))
(mkdir-p root)
(let* ((user (getpwnam user))
(group (getpwnam group)))
(for-each
(lambda (dir)
(mkdir-p (string-append root "/" dir)))
'("guix" "ssh" "home" "celery" "backups" "cache" "cache/fonts"))
(for-each
(lambda (file)
;; nginx needs to serve static files
(chmod file #o750)
(chown file (passwd:uid user) (passwd:gid group)))
(find-files root #:directories? #t))))
;; Delete stale pid files
(let ((pid-dir (string-append root "/celery/pids"))
(user (getpwnam user))
(group (getpwnam group)))
(when (file-exists? pid-dir)
(delete-file-recursively pid-dir))
(mkdir pid-dir)
(chmod pid-dir #o750)
(chown pid-dir (passwd:uid user) (passwd:gid group)))
(let ((pid-dir (string-append root "/home"))
(user (getpwnam user))
(group (getpwnam group)))
(when (file-exists? pid-dir)
(delete-file-recursively pid-dir))
(mkdir pid-dir)
(chmod pid-dir #o750)
(chown pid-dir (passwd:uid user) (passwd:gid group)))
(setenv "DATA_DIR" root)
(let ((guix-dir (string-append root "/guix"))
(user (getpwnam user))
(group (getpwnam group)))
(when (file-exists? guix-dir)
(delete-file-recursively guix-dir))
(mkdir guix-dir)
(call-with-output-file (string-append root "/guix/__init__.py")
(const #t))
(chmod guix-dir #o750)
(chmod (string-append root "/guix/__init__.py") #o750)
(chown guix-dir (passwd:uid user) (passwd:gid group))
;; TODO: This is copied from the touch implementation somewhere
(copy-file settings-file (string-append root "/guix/settings.py")))
(copy-file
#$(file-append python-weblate "/lib/python"
(version-major+minor (package-version python))
"/site-packages/weblate/examples/weblate.uwsgi.ini")
#$(string-append (weblate-configuration-root config) "/uwsgi.ini"))
(substitute* #$(string-append (weblate-configuration-root config) "/uwsgi.ini")
;; TODO: Change this too
;; uid = weblate
;; gid = weblate
(("socket\\s+=.*" all)
(string-append "socket = "
#$(weblate-configuration-uwsgi-listen config)
"\n")))
(delete-file (string-append root "/guix/settings.py"))
(copy-file #$(weblate-configuration-settings-file config)
(string-append root "/guix/settings.py"))
(let ((user (getpwnam user))
(group (getpwnam group)))
(for-each
(lambda (file)
(let ((file (string-append
#$(weblate-configuration-root config)
file)))
(chmod file #o750)
(chown file (passwd:uid user) (passwd:gid group))))
'("/guix/settings.py" "/guix/__init__.py" "/uwsgi.ini")))
(setenv "PYTHONPATH" #$(weblate-configuration-root config))
(setenv "DJANGO_SETTINGS_MODULE" "guix.settings")
(setenv "GI_TYPELIB_PATH"
;; FIXE: use the correct path
"/run/current-system/profile/lib/girepository-1.0")
(let ((pid (primitive-fork))
(weblate (getpwnam "weblate")))
(if (eq? pid 0)
(dynamic-wind
(const #t)
(lambda ()
(setgid (passwd:gid weblate))
(setuid (passwd:uid weblate))
(primitive-exit
(if (system*
#$(file-append python-weblate "/bin/weblate")
"collectstatic" "--noinput")
0
1)))
(lambda ()
(primitive-exit 1)))
(zero? (cdr (waitpid pid)))))
(let ((user (getpwnam user))
(group (getpwnam group)))
(for-each
(lambda (file)
;; nginx needs to serve static files
(chmod file #o750)
(chown file (passwd:uid user) (passwd:gid group)))
(find-files "static" #:directories? #t))
;; Be sure that private key permissions are right
(let ((private-key (string-append root "/ssh/id_rsa")))
(chmod private-key #o600)
(chown private-key (passwd:uid user) (passwd:gid group)))))))
(define (weblate-nginx-service config)
(let ((listen (weblate-configuration-listen config))
(root (weblate-configuration-root config)))
(list
(nginx-server-configuration
(listen listen)
(server-name '("weblate"))
(root root)
(locations
(list
(nginx-location-configuration
(uri "~ ^/favicon.ico$")
(body `(,(string-append "alias " root "/static/favicon.ico;")
"expires 30d;")))
(nginx-location-configuration
(uri "/static/")
(body `(,(string-append "alias " root "/static/;")
"expires 30d;")))
(nginx-location-configuration
(uri "/media/")
(body `(,(string-append "alias " root "/media/;")
"expires 30d;")))
(nginx-location-configuration
(uri "/")
(body
`(,#~(string-append
"include " '#$nginx
"/share/nginx/conf/uwsgi_params;")
"# Needed for long running operations in"
"# admin interface"
"uwsgi_read_timeout 3600;"
"# Adjust based to uwsgi configuration:"
"# uwsgi_pass unix:///run/uwsgi/app/weblate/socket;"
,(string-append "uwsgi_pass "
(weblate-configuration-uwsgi-listen config)
";"))))))
(try-files (list "$uri" "$uri/index.html"))))))
(define (uwsgi-weblate-service config)
(shepherd-service
(documentation "Run uwsgi weblate service.")
(provision '(weblate-uwsgi))
(requirement '(networking))
(start #~(make-forkexec-constructor
'(#$(file-append uwsgi "/bin/uwsgi")
#$(string-append (weblate-configuration-root config) "/uwsgi.ini"))
#:environment-variables
`(,(string-append "PYTHONPATH="
#$(weblate-configuration-root config))
"DJANGO_SETTINGS_MODULE=guix.settings"
"GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt"
"LC_ALL=en_US.utf8" "LANG=en_US.utf8"
"GI_TYPELIB_PATH=/run/current-system/profile/lib/girepository-1.0")
;; ;; #:environment-variables
;; (list (string-append "PYTHON_PATH=" root))
#:user #$(weblate-configuration-user config)))
(stop #~(make-kill-destructor))))
;; FIXME: this is an hack to get a celery program that knows where to
;; find weblate dependencies
(define python-celery-for-weblate
(package
(inherit python-celery)
(propagated-inputs
(list
(append
(package-inputs python-celery)
`("python-weblate" ,python-weblate))))))
(define (celery-weblate-service config)
(shepherd-service
(documentation "Run celery weblate service. Restart with RESTART.")
(provision '(weblate-celery))
;; TODO: abstract the celery call and put it here according to
;; https://docs.weblate.org/en/latest/admin/install.html#background-tasks-using-celery
;; (actions '((shepherd-action
;; (name 'reststart)
;; (documentation "Restart celery runners")
;; (procedure #~(lambda (running . args)
;; (format #t "Hello, friend! arguments: ~s\n"
;; args)
;; #t)))))
(requirement '(networking)) ;redis?
(auto-start? #t)
(start #~(lambda _ ;; celery is forking
(let ((pid (primitive-fork))
(weblate (getpwnam "weblate")))
(if (eq? pid 0)
(dynamic-wind
(const #t)
(lambda ()
(setgid (passwd:gid weblate))
(setuid (passwd:uid weblate))
(setenv "GIT_SSL_CAINFO" "/etc/ssl/certs/ca-certificates.crt")
(primitive-exit
(if (zero?
(system*
#$(file-append python-celery-for-weblate "/bin/celery")
"multi" "start"
;; Processes to start
"celery" "notify" "backup" "memory" "translate"
;; must be under a folder which is writable by weblate!
(string-append
"--pidfile="
#$(weblate-configuration-root config)
"/celery/pids/weblate-%n.pid")
(string-append
"--logfile="
#$(weblate-configuration-root config)
"/celery/weblate-%n%I.log")
;; FIXME: pass as param!
(string-append "--loglevel=" "INFO")
"-A" "weblate.utils"
"--beat:celery"
"--queues:celery=celery"
"--prefetch-multiplier:celery=4"
"--queues:notify=notify"
"--prefetch-multiplier:notify=10"
"--queues:memory=memory"
"--prefetch-multiplier:memory=10"
"--queues:translate=translate"
"--prefetch-multiplier:translate=4"
"--concurrency:backup=1"
"--queues:backup=backup"
"--prefetch-multiplier:backup=2"))
0
1)))
(lambda ()
(primitive-exit 1)))
(zero? (cdr (waitpid pid)))))
#:environment-variables
;; FIXME: use python-weblate store path instead of this
`(,(string-append
"PYTHONPATH="
(string-join
(list
;; for settings path
#$(weblate-configuration-root config)
;; FIXME: Replace the hack above
;; (python-celery-for-weblate) with the right list
;; of search paths. This include weblate path and
;; all weblate dependencies
)
":"))
"GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt"
"DJANGO_SETTINGS_MODULE=guix.settings"
;; Internal Weblate variable to indicate we're running inside Celery
"CELERY_WORKER_RUNNING=1")
;; #:environment-variables
;; '(
;;
;; "LC_ALL=en_US.utf8" "LANG=en_US.utf8"
;; "GI_TYPELIB_PATH=/run/current-system/profile/lib/girepository-1.0")
;; ;; #:environment-variables
;; (list (string-append "PYTHON_PATH=" root))
#:user #$(weblate-configuration-user config)
#:group #$(weblate-configuration-group config)))
(stop #~(make-kill-destructor))))

1570
weblate.scm Normal file

File diff suppressed because it is too large Load Diff