diff --git a/10.0-nightly/Dockerfile b/10.0-nightly/Dockerfile index 490bc484c..8d0efa781 100644 --- a/10.0-nightly/Dockerfile +++ b/10.0-nightly/Dockerfile @@ -3,18 +3,23 @@ LABEL maintainer="Archeti " # Generate locale C.UTF-8 for postgres and general locale data ENV LANG C.UTF-8 +ARG DEBIAN_FRONTEND=noninteractive +COPY ./platform.sh /platform.sh # Install some dependencies python3.7 RUN set -x; \ - apt-get update \ + export ARCH=`sh /platform.sh` \ + # Setup deb source for postgresql client + && apt-get update \ && apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg \ python-wheel \ python-setuptools \ python-pip \ python2.7 \ libpython2.7 \ curl \ - gnupg \ libpq-dev \ libsasl2-2 \ libldap-2.4-2 \ @@ -23,28 +28,26 @@ RUN set -x; \ libxslt1.1 \ sudo \ node-less \ - # python-yaml \ - && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb \ - && apt-get install -y --no-install-recommends ./wkhtmltox.deb \ + && echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ + && export GNUPGHOME="$(mktemp -d)" \ + && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ + && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ + && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ + && gpgconf --kill all \ + && rm -rf "$GNUPGHOME" \ + # Download deb for wkhtmltox + && echo "Downloading https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb" \ + && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + ./wkhtmltox.deb \ && rm -rf /var/lib/apt/lists/* wkhtmltox.deb -# Install latest postgresql-client -RUN set -x; \ - echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ - && export GNUPGHOME="$(mktemp -d)" \ - && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ - && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ - && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ - && gpgconf --kill all \ - && rm -rf "$GNUPGHOME" \ - && apt-get update \ - && apt-get install -y postgresql-client \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH=/usr/local/bin:$PATH +ENV PATH=/var/lib/odoo/.local/bin:/usr/local/bin:$PATH # Install Odoo Including things from sources ENV ODOO_VERSION 10.0 -ENV ODOO_RELEASE=20210625 +ENV ODOO_RELEASE= ARG ODOO_ARCHIVE=odoo_${ODOO_VERSION}.${ODOO_RELEASE}.tar.gz RUN set -x; \ apt-get update \ @@ -56,15 +59,14 @@ RUN set -x; \ libxml2-dev \ libxmlsec1-dev \ libxslt1-dev \ + git \ && pip install -U pip \ && /usr/bin/env pip install \ - psycogreen \ + odoo-tools==0.0.65 \ \ - && curl -o requirements.txt https://raw.githubusercontent.com/odoo/odoo/${ODOO_VERSION}/requirements.txt \ - && pip install -U pip \ - && pip install -r requirements.txt \ - && /usr/bin/env pip install https://nightly.odoo.com/${ODOO_VERSION}/nightly/src/${ODOO_ARCHIVE} \ - && cd / \ + && odoo-install --release "" --version "10.0" --repo "https://github.com/odoo/odoo.git" --ref "" \ + && [ -f /usr/local/bin/odoo.py ] && ln -s /usr/local/bin/odoo.py /usr/local/bin/odoo \ + || cd / \ && apt-get --purge remove -y \ build-essential \ python2.7-dev \ @@ -73,17 +75,14 @@ RUN set -x; \ libxml2-dev \ libxmlsec1-dev \ libxslt1-dev \ + git \ && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* - -VOLUME /etc/odoo -VOLUME /var/lib/odoo + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /root/.cache COPY ./odoo.conf /etc/odoo/ COPY ./entrypoint.py / COPY ./sudo-entrypoint.py / -COPY ./find_modules.py /scripts/find_modules.py -COPY ./prepare_project.py /scripts/prepare_project.py ARG UID=1000 ARG GID=1000 @@ -93,12 +92,13 @@ RUN mkdir /addons \ && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ && chown odoo /etc/odoo/odoo.conf \ && chown -R odoo:odoo /addons \ - && cp /usr/local/bin/odoo.py /usr/local/bin/odoo || true \ && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /sudo-entrypoint.py" >> /etc/sudoers -ENV OPENERP_SERVER /etc/odoo/odoo.conf +VOLUME /etc/odoo +VOLUME /var/lib/odoo + ENV ODOO_RC /etc/odoo/odoo.conf -ENV ODOO_BASE_PATH /usr/local/lib/python2.7/dist-packages/odoo/addons +ENV ODOO_BASE_PATH /usr/local/lib/python2.7/dist-packages/openerp/addons # Env variable defined to monitor the kind of service running # it could be a staging/production/test or anything and undefined # is the default in case we need to know servers that aren't correctly @@ -111,15 +111,15 @@ expose 8071 USER odoo LABEL version="10.0" -LABEL release="20210625" +LABEL release="" -LABEL org.opencontainers.image.created="2021-06-25T17:55:41.198545" +LABEL org.opencontainers.image.created="2022-11-04T20:06:13.821080" LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" -LABEL org.opencontainers.image.authors="Archeti " +LABEL org.opencontainers.image.authors="OdooPlus " LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.version="10.0" -LABEL org.opencontainers.image.vendor="ArcheTI" +LABEL org.opencontainers.image.vendor="OdooPlus" LABEL org.opencontainers.image.ref.name="10.0-nightly" LABEL org.opencontainers.image.title="Odoo 10.0" LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." diff --git a/10.0-nightly/entrypoint.py b/10.0-nightly/entrypoint.py index f2ce4654b..bb0ffddc3 100755 --- a/10.0-nightly/entrypoint.py +++ b/10.0-nightly/entrypoint.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # import argparse +from __future__ import print_function import time import shlex import subprocess @@ -17,25 +18,9 @@ from passlib.context import CryptContext from contextlib import contextmanager -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - -try: - from pip._internal.network.session import PipSession - from pip._internal.req.req_file import parse_requirements - from pip._internal.req.constructors import ( - install_req_from_parsed_requirement - ) -except Exception: - from pip.download import PipSession - from pip.req.req_file import parse_requirements - - def install_req_from_parsed_requirement(req): - return req - -from collections import defaultdict +from odoo_tools.compat import Path +from odoo_tools.modules.search import find_addons_paths, find_modules_paths +from odoo_tools.configuration.pips import merge_requirements try: # python3 @@ -68,63 +53,6 @@ def quote(s): return "'" + s.replace("'", "'\"'\"'") + "'" -class Requirement(object): - def __init__(self): - self.extras = set() - self.specifiers = set() - self.links = set() - self.editable = False - - -def merge_requirements(files): - requirements = defaultdict(lambda: Requirement()) - links = set() - - for filename in files: - f_requirements = parse_requirements( - str(filename), session=PipSession() - ) - for parsed_requirement in f_requirements: - requirement = install_req_from_parsed_requirement( - parsed_requirement - ) - - if requirement.markers and not requirement.markers.evaluate(): - continue - - if not hasattr(requirement.req, 'name'): - links.add(requirement.link.url) - break - name = requirement.req.name - specifiers = requirement.req.specifier - extras = requirement.req.extras - requirements[name].extras |= set(extras) - requirements[name].specifiers |= set(specifiers) - if requirement.link: - requirements[name].links |= {requirement.link.url} - requirements[name].editable |= requirement.editable - - result = [] - for key, value in requirements.items(): - if value.links: - result.append("%s" % value.links.pop()) - elif not value.extras: - result.append( - "%s %s" % (key, ",".join(map(str, value.specifiers))) - ) - else: - result.append("%s [%s] %s" % ( - key, - ",".join(map(str, value.extras)), - ",".join(map(str, value.specifiers)) - )) - - for link in links: - result.append(link) - - return "\n".join(result) - - def pipe(args): """ Call the process with std(in,out,err) @@ -178,42 +106,12 @@ def call_sudo_entrypoint(): return ret -def get_all_manifests(): - odoo_manifests = glob.glob('/addons/**/__manifest__.py', recursive=True) - erp_manifest = glob.glob('/addons/**/__openerp__.py', recursive=True) - - return odoo_manifests + erp_manifest - - -def module_name(manifest_file): - return path.basename(path.dirname(manifest_file)) - - -def get_module(manifest_file): - with open(manifest_file) as manifest_in: - try: - code = compile(manifest_in.read(), '__manifest__', 'eval') - module = eval(code, {}, {}) - except Exception: - module = {} - - return module - - -def is_installable(manifest): - return manifest.get('installable', True) - - -def is_server_wide(manifest): - return manifest.get('server_wide', False) - - def get_server_wide_modules(manifests): base_server_wide_modules = ['base', 'web'] custom_server_wide_modules = [ - module_name(filename) - for filename, module in manifests.items() - if is_server_wide(module) + manifest.technical_name + for manifest in manifests + if manifest.server_wide ] return base_server_wide_modules + custom_server_wide_modules @@ -246,7 +144,12 @@ def install_python_dependencies(valid_paths): req_file = Path('/var/lib/odoo/requirements.txt') with req_file.open('w') as fout: data = merge_requirements(requirement_files) - fout.write(data) + data = "\n".join(data) + + try: + fout.write(data) + except Exception: + fout.write(data.decode('utf-8')) for requirements in requirement_files: print("Installing python packages from %s" % requirements) @@ -338,65 +241,44 @@ def get_extra_paths(): ] -def get_valid_paths(): - base_addons = os.environ.get('ODOO_BASE_PATH') +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] - # addons = os.listdir('/addons') - valid_paths = [base_addons] +def get_valid_paths(): + base_addons = os.environ['ODOO_BASE_PATH'] - addons_paths = get_dirs('/addons') - addons_paths += get_extra_paths() + base_addons_paths = [base_addons, '/addons'] + base_addons_paths += get_extra_paths() - for addons_path in addons_paths: - print("Lookup addons in %s" % addons_path) - flush_streams() - addons = get_dirs(addons_path) - for addon in addons: - files = os.listdir(addon) - if ( - '__init__.py' in files and - '__manifest__.py' in files or - '__openerp__.py' in files - ): - valid_paths.append(addons_path) - print("Addons found !") - flush_streams() - break - if addons_path in valid_paths: - continue - else: - print("No addons found in path. Skipping...") - flush_streams() + valid_paths = find_addons_paths(base_addons_paths) + + for valid_path in valid_paths: + print("Found installable modules in {}".format(valid_path)) return valid_paths def setup_addons_paths(config, valid_paths): - config.set('options', 'addons_path', ",".join(valid_paths)) + config.set('options', 'addons_path', ",".join([ + str(path) for path in valid_paths + ])) flush_streams() -def setup_server_wide_modules(config): +def setup_server_wide_modules(config, valid_paths): print("Searching for server wide modules") - all_manifests = get_all_manifests() + filters = set(['installable']) + all_manifests = find_modules_paths(valid_paths, filters) - print("Found %s modules" % (len(all_manifests))) + print("Found %s installable modules " % (len(all_manifests),)) - manifests = { - filename: get_module(filename) - for filename in all_manifests - } - - installable_manifests = { - filename: module - for filename, module in manifests.items() - if is_installable(module) - } - - print("Found %s installable modules " % (len(installable_manifests),)) - - server_wide_modules = get_server_wide_modules(installable_manifests) + server_wide_modules = get_server_wide_modules(all_manifests) if len(server_wide_modules) > 2: modules = ",".join(server_wide_modules) @@ -408,6 +290,36 @@ def setup_server_wide_modules(config): flush_streams() +def convert_value(name, value): + if value in ['True', 'False']: + return value == 'True' + + return value + + +def setup_env_config(config): + try: + from odoo.tools import config as odoo_config + except ImportError: + from openerp.tools import config as odoo_config + + params_by_name = {} + for key, opt in odoo_config.casts.items(): + value_opt = opt.get_opt_string() + value_opt = value_opt.upper().replace('--', 'ODOO_') + value_opt = value_opt.replace('-', '_') + params_by_name[value_opt] = key + + for key, value in os.environ.items(): + if not key.startswith('ODOO_'): + continue + + if key in params_by_name: + config_name = params_by_name[key] + converted_value = convert_value(key, value) + config.set('options', config_name, converted_value) + + def setup_environ(config): print("Configuring environment variables for postgresql") @@ -499,12 +411,39 @@ def wait_postgresql(): @contextmanager def get_config(config_path): config = ConfigParser() - config.read(config_path) + config.read(str(config_path)) yield config - with config_path.open('w') as out: - config.write(out) + try: + with config_path.open('w') as out: + config.write(out) + except Exception: + with config_path.open('wb') as out: + config.write(out) + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for vpath in paths: + cur_path = Path(vpath) + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(vpath) + + return res_paths def main(): @@ -518,7 +457,9 @@ def main(): sys.exit(ret) # Install python packages with pip in user home - valid_paths = get_valid_paths() + orig_valid_paths = get_valid_paths() + excluded_paths = get_excluded_paths() + valid_paths = filter_valid_paths(orig_valid_paths, excluded_paths) config_path = Path(os.environ.get('ODOO_RC', '/etc/odoo/odoo.cfg')) @@ -528,7 +469,8 @@ def main(): install_master_password(config) setup_environ(config) setup_addons_paths(config, valid_paths) - setup_server_wide_modules(config) + setup_server_wide_modules(config, valid_paths) + setup_env_config(config) if not os.environ.get('ODOO_SKIP_POSTGRES_WAIT'): wait_postgresql() diff --git a/10.0-nightly/find_modules.py b/10.0-nightly/find_modules.py deleted file mode 100644 index 9cb943748..000000000 --- a/10.0-nightly/find_modules.py +++ /dev/null @@ -1,76 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -from itertools import chain - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - "-p", - dest='paths', - action="append", - help="Location in which to search", - default=[] - ) - - parser.add_option( - "--only-name", - dest="only_name", - action="store_true", - help="Only display module name instead of path", - default=False, - ) - - parser.add_option( - '--csv', - dest="is_csv", - action="store_true", - help="Output as a comma separated list", - default=False - ) - - return parser - - -def find_modules(options, path): - modules = set() - - path = Path.cwd() / path - - erp_manifest = '__openerp__.py' - odoo_manifest = '__manifest__.py' - - manifest_globs = chain( - path.glob('**/{}'.format(erp_manifest)), - path.glob('**/{}'.format(odoo_manifest)), - ) - - for path in manifest_globs: - rel_path = path.parent.relative_to(Path.cwd()) - if options.only_name: - modules.add(rel_path.name) - else: - modules.add(str(rel_path)) - - return modules - - -def main(options, args): - modules = set() - for path in options.paths: - modules = modules.union(find_modules(options, path)) - - return modules - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - modules = main(options, args) - - if not options.is_csv: - for module in modules: - print(module) - else: - print(",".join(modules), end="") diff --git a/10.0-nightly/platform.sh b/10.0-nightly/platform.sh new file mode 100644 index 000000000..fdf5e42eb --- /dev/null +++ b/10.0-nightly/platform.sh @@ -0,0 +1,13 @@ +PLATFORM=`uname -m` + +if [ $PLATFORM = "aarch64" ] +then + echo -n "arm64" +else +if [ $PLATFORM = "x86_64" ] +then + echo -n "amd64" +else + echo -n $PLATFORM +fi +fi diff --git a/10.0-nightly/prepare_project.py b/10.0-nightly/prepare_project.py deleted file mode 100644 index 74a53f2e0..000000000 --- a/10.0-nightly/prepare_project.py +++ /dev/null @@ -1,198 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -import toml -from giturlparse import parse -from contextlib import contextmanager -import os -from subprocess import run -from urllib.parse import urlparse - - -def get_services(services): - return { - service.get('name'): service - for service in services.get('services') - } - - -def addons_by_project(options, addons): - res = {} - - for addon in addons: - if addon['url'] == 'self': - if options.ignore_self: - continue - parsed = parse(options.url) - else: - parsed = parse(addon['url']) - - auth = parsed.protocol in ['git', 'ssh'] - - res[parsed.repo] = dict( - addon, - url=parsed.url2https, - auth=auth - ) - - return res - - -def merge_addons(options, base, other): - base_addons = addons_by_project(options, base) - other_addons = addons_by_project(options, other) - - for name, addon in other_addons.items(): - if name not in base_addons: - base_addons[name] = addon - else: - base_addons[name] = dict(base_addons[name], **addon) - - return [ - addon - for addon in base_addons.values() - ] - - -def merge_services(options, base, other): - basic_inherit = dict(base, **other) - - if base.get('addons') or other.get('addons'): - basic_inherit['addons'] = merge_addons( - options, - base.get('addons', []), - other.get('addons', []) - ) - - return basic_inherit - - -def compile_service(options, services, name): - service = services.get(name, {}) - if 'inherit' in service: - merge_service = compile_service(options, services, service['inherit']) - service = merge_services(options, merge_service, service) - - return service - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - '-f', - '--file', - dest="file", - help="Input File" - ) - - parser.add_option( - '--url', - dest='url', - help="Url of self project" - ) - - parser.add_option( - '-o', - '--output', - dest="output_directory", - help="Output Directory" - ) - - parser.add_option( - '-e', - dest="env", - help="Environment to prepare" - ) - - parser.add_option( - '--username', - dest="username", - help="Username to replace with", - ) - - parser.add_option( - '--password', - dest="password", - help="password to set on https urls" - ) - - parser.add_option( - '-b', - '--branch', - dest="branch", - help="Default branch if no ref is defined" - ) - - parser.add_option( - '--ignore-self', - dest="ignore_self", - action="store_true", - help="Ignore self url as it's already fetched", - default=False - ) - - return parser - - -@contextmanager -def cd(directory): - cwd = Path.cwd() - try: - os.chdir(directory) - yield - finally: - os.chdir(cwd) - - -def fetch_addons(options, addon): - parsed = parse(addon['url']) - - url = urlparse(parsed.url2https) - - if addon['auth'] and options.username and options.password: - url = url._replace( - netloc="{}:{}@{}".format( - options.username, - options.password, - url.netloc - ) - ) - - repo_path = Path.cwd() / options.output_directory / parsed.repo - - repo_path.mkdir(exist_ok=True) - - with cd(repo_path): - run(['git', 'init']) - run(['git', 'remote', 'add', 'origin', url.geturl()]) - - ref = addon.get('commit') or addon.get('branch') or options.branch - - if ref: - run(['git', 'fetch', 'origin', ref]) - else: - run(['git', 'fetch', 'origin']) - - run(['git', 'checkout', 'FETCH_HEAD']) - run(['git', 'remote', 'remove', 'origin']) - - -def main(options, args): - with Path(options.file).open('r') as fin: - services = toml.loads(fin.read()) - - by_name = get_services(services) - - outputdir = Path(options.output_directory) - outputdir.mkdir(exist_ok=True) - - service = compile_service(options, by_name, options.env) - - for addons in service.get('addons', []): - fetch_addons(options, addons) - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - main(options, args) diff --git a/10.0-nightly/sudo-entrypoint.py b/10.0-nightly/sudo-entrypoint.py index 98779baae..916239170 100755 --- a/10.0-nightly/sudo-entrypoint.py +++ b/10.0-nightly/sudo-entrypoint.py @@ -12,6 +12,7 @@ from os import path from os.path import expanduser import shutil +import six try: from pathlib import Path @@ -69,6 +70,37 @@ def get_addons_paths(): ] +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for cur_path in paths: + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(cur_path) + + return res_paths + + def install_apt_packages(): """ Install debian dependencies. @@ -76,26 +108,34 @@ def install_apt_packages(): package_list = set() paths = get_addons_paths() + excluded_paths = get_excluded_paths() + paths = filter_valid_paths(paths, excluded_paths) print("Looking up for packages in {}".format(paths)) for addons_path in paths: for packages in addons_path.glob('**/apt-packages.txt'): print("Installing packages from %s" % packages) - with open(packages, 'r') as pack_file: - lines = [line.strip() for line in pack_file] + with packages.open('r') as pack_file: + lines = [ + six.ensure_text(line).strip() + for line in pack_file + if six.ensure_text(line).strip() + ] package_list.update(set(lines)) extras = os.environ.get('EXTRA_APT_PACKAGES', '') - print(f"Adding extra packages {extras}") + print("Adding extra packages {extras}".format(extras=extras)) if extras: for package in extras.split(','): - if not package: + if not package.strip(): continue - package_list.add(package) + package_list.add( + six.ensure_text(package).strip() + ) if len(package_list) > 0: - print(f"Installing {package_list}") + print("Installing {package_list}".format(package_list=package_list)) ret = pipe(['apt-get', 'update']) # Something went wrong, stop the service as it's failing diff --git a/10.0/Dockerfile b/10.0/Dockerfile index 578120f30..45229eb91 100644 --- a/10.0/Dockerfile +++ b/10.0/Dockerfile @@ -3,18 +3,23 @@ LABEL maintainer="Archeti " # Generate locale C.UTF-8 for postgres and general locale data ENV LANG C.UTF-8 +ARG DEBIAN_FRONTEND=noninteractive +COPY ./platform.sh /platform.sh # Install some dependencies python3.7 RUN set -x; \ - apt-get update \ + export ARCH=`sh /platform.sh` \ + # Setup deb source for postgresql client + && apt-get update \ && apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg \ python-wheel \ python-setuptools \ python-pip \ python2.7 \ libpython2.7 \ curl \ - gnupg \ libpq-dev \ libsasl2-2 \ libldap-2.4-2 \ @@ -23,28 +28,26 @@ RUN set -x; \ libxslt1.1 \ sudo \ node-less \ - # python-yaml \ - && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb \ - && apt-get install -y --no-install-recommends ./wkhtmltox.deb \ + && echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ + && export GNUPGHOME="$(mktemp -d)" \ + && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ + && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ + && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ + && gpgconf --kill all \ + && rm -rf "$GNUPGHOME" \ + # Download deb for wkhtmltox + && echo "Downloading https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb" \ + && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + ./wkhtmltox.deb \ && rm -rf /var/lib/apt/lists/* wkhtmltox.deb -# Install latest postgresql-client -RUN set -x; \ - echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ - && export GNUPGHOME="$(mktemp -d)" \ - && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ - && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ - && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ - && gpgconf --kill all \ - && rm -rf "$GNUPGHOME" \ - && apt-get update \ - && apt-get install -y postgresql-client \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH=/usr/local/bin:$PATH +ENV PATH=/var/lib/odoo/.local/bin:/usr/local/bin:$PATH # Install Odoo Including things from sources ENV ODOO_VERSION 10.0 -ENV ODOO_RELEASE=20200313 +ENV ODOO_RELEASE=20201204 ARG ODOO_ARCHIVE=odoo_${ODOO_VERSION}.${ODOO_RELEASE}.tar.gz RUN set -x; \ apt-get update \ @@ -56,15 +59,14 @@ RUN set -x; \ libxml2-dev \ libxmlsec1-dev \ libxslt1-dev \ + git \ && pip install -U pip \ && /usr/bin/env pip install \ - psycogreen \ + odoo-tools==0.0.65 \ pathlib2 \ - && curl -o requirements.txt https://raw.githubusercontent.com/odoo/odoo/${ODOO_VERSION}/requirements.txt \ - && pip install -U pip \ - && pip install -r requirements.txt \ - && /usr/bin/env pip install https://nightly.odoo.com/${ODOO_VERSION}/nightly/src/${ODOO_ARCHIVE} \ - && cd / \ + && odoo-install --release "20201204" --version "10.0" --repo "https://github.com/odoo/odoo.git" --ref "" \ + && [ -f /usr/local/bin/odoo.py ] && ln -s /usr/local/bin/odoo.py /usr/local/bin/odoo \ + || cd / \ && apt-get --purge remove -y \ build-essential \ python2.7-dev \ @@ -73,17 +75,14 @@ RUN set -x; \ libxml2-dev \ libxmlsec1-dev \ libxslt1-dev \ + git \ && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* - -VOLUME /etc/odoo -VOLUME /var/lib/odoo + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /root/.cache COPY ./odoo.conf /etc/odoo/ COPY ./entrypoint.py / COPY ./sudo-entrypoint.py / -COPY ./find_modules.py /scripts/find_modules.py -COPY ./prepare_project.py /scripts/prepare_project.py ARG UID=1000 ARG GID=1000 @@ -93,12 +92,13 @@ RUN mkdir /addons \ && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ && chown odoo /etc/odoo/odoo.conf \ && chown -R odoo:odoo /addons \ - && cp /usr/local/bin/odoo.py /usr/local/bin/odoo || true \ && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /sudo-entrypoint.py" >> /etc/sudoers -ENV OPENERP_SERVER /etc/odoo/odoo.conf +VOLUME /etc/odoo +VOLUME /var/lib/odoo + ENV ODOO_RC /etc/odoo/odoo.conf -ENV ODOO_BASE_PATH /usr/local/lib/python2.7/dist-packages/odoo/addons +ENV ODOO_BASE_PATH /usr/local/lib/python2.7/dist-packages/openerp/addons # Env variable defined to monitor the kind of service running # it could be a staging/production/test or anything and undefined # is the default in case we need to know servers that aren't correctly @@ -111,15 +111,15 @@ expose 8071 USER odoo LABEL version="10.0" -LABEL release="20200313" +LABEL release="20201204" -LABEL org.opencontainers.image.created="2021-06-25T17:55:41.193896" +LABEL org.opencontainers.image.created="2022-11-04T20:06:13.799940" LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" -LABEL org.opencontainers.image.authors="Archeti " +LABEL org.opencontainers.image.authors="OdooPlus " LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.version="10.0" -LABEL org.opencontainers.image.vendor="ArcheTI" +LABEL org.opencontainers.image.vendor="OdooPlus" LABEL org.opencontainers.image.ref.name="10.0" LABEL org.opencontainers.image.title="Odoo 10.0" LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." diff --git a/10.0/entrypoint.py b/10.0/entrypoint.py index f2ce4654b..bb0ffddc3 100755 --- a/10.0/entrypoint.py +++ b/10.0/entrypoint.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # import argparse +from __future__ import print_function import time import shlex import subprocess @@ -17,25 +18,9 @@ from passlib.context import CryptContext from contextlib import contextmanager -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - -try: - from pip._internal.network.session import PipSession - from pip._internal.req.req_file import parse_requirements - from pip._internal.req.constructors import ( - install_req_from_parsed_requirement - ) -except Exception: - from pip.download import PipSession - from pip.req.req_file import parse_requirements - - def install_req_from_parsed_requirement(req): - return req - -from collections import defaultdict +from odoo_tools.compat import Path +from odoo_tools.modules.search import find_addons_paths, find_modules_paths +from odoo_tools.configuration.pips import merge_requirements try: # python3 @@ -68,63 +53,6 @@ def quote(s): return "'" + s.replace("'", "'\"'\"'") + "'" -class Requirement(object): - def __init__(self): - self.extras = set() - self.specifiers = set() - self.links = set() - self.editable = False - - -def merge_requirements(files): - requirements = defaultdict(lambda: Requirement()) - links = set() - - for filename in files: - f_requirements = parse_requirements( - str(filename), session=PipSession() - ) - for parsed_requirement in f_requirements: - requirement = install_req_from_parsed_requirement( - parsed_requirement - ) - - if requirement.markers and not requirement.markers.evaluate(): - continue - - if not hasattr(requirement.req, 'name'): - links.add(requirement.link.url) - break - name = requirement.req.name - specifiers = requirement.req.specifier - extras = requirement.req.extras - requirements[name].extras |= set(extras) - requirements[name].specifiers |= set(specifiers) - if requirement.link: - requirements[name].links |= {requirement.link.url} - requirements[name].editable |= requirement.editable - - result = [] - for key, value in requirements.items(): - if value.links: - result.append("%s" % value.links.pop()) - elif not value.extras: - result.append( - "%s %s" % (key, ",".join(map(str, value.specifiers))) - ) - else: - result.append("%s [%s] %s" % ( - key, - ",".join(map(str, value.extras)), - ",".join(map(str, value.specifiers)) - )) - - for link in links: - result.append(link) - - return "\n".join(result) - - def pipe(args): """ Call the process with std(in,out,err) @@ -178,42 +106,12 @@ def call_sudo_entrypoint(): return ret -def get_all_manifests(): - odoo_manifests = glob.glob('/addons/**/__manifest__.py', recursive=True) - erp_manifest = glob.glob('/addons/**/__openerp__.py', recursive=True) - - return odoo_manifests + erp_manifest - - -def module_name(manifest_file): - return path.basename(path.dirname(manifest_file)) - - -def get_module(manifest_file): - with open(manifest_file) as manifest_in: - try: - code = compile(manifest_in.read(), '__manifest__', 'eval') - module = eval(code, {}, {}) - except Exception: - module = {} - - return module - - -def is_installable(manifest): - return manifest.get('installable', True) - - -def is_server_wide(manifest): - return manifest.get('server_wide', False) - - def get_server_wide_modules(manifests): base_server_wide_modules = ['base', 'web'] custom_server_wide_modules = [ - module_name(filename) - for filename, module in manifests.items() - if is_server_wide(module) + manifest.technical_name + for manifest in manifests + if manifest.server_wide ] return base_server_wide_modules + custom_server_wide_modules @@ -246,7 +144,12 @@ def install_python_dependencies(valid_paths): req_file = Path('/var/lib/odoo/requirements.txt') with req_file.open('w') as fout: data = merge_requirements(requirement_files) - fout.write(data) + data = "\n".join(data) + + try: + fout.write(data) + except Exception: + fout.write(data.decode('utf-8')) for requirements in requirement_files: print("Installing python packages from %s" % requirements) @@ -338,65 +241,44 @@ def get_extra_paths(): ] -def get_valid_paths(): - base_addons = os.environ.get('ODOO_BASE_PATH') +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] - # addons = os.listdir('/addons') - valid_paths = [base_addons] +def get_valid_paths(): + base_addons = os.environ['ODOO_BASE_PATH'] - addons_paths = get_dirs('/addons') - addons_paths += get_extra_paths() + base_addons_paths = [base_addons, '/addons'] + base_addons_paths += get_extra_paths() - for addons_path in addons_paths: - print("Lookup addons in %s" % addons_path) - flush_streams() - addons = get_dirs(addons_path) - for addon in addons: - files = os.listdir(addon) - if ( - '__init__.py' in files and - '__manifest__.py' in files or - '__openerp__.py' in files - ): - valid_paths.append(addons_path) - print("Addons found !") - flush_streams() - break - if addons_path in valid_paths: - continue - else: - print("No addons found in path. Skipping...") - flush_streams() + valid_paths = find_addons_paths(base_addons_paths) + + for valid_path in valid_paths: + print("Found installable modules in {}".format(valid_path)) return valid_paths def setup_addons_paths(config, valid_paths): - config.set('options', 'addons_path', ",".join(valid_paths)) + config.set('options', 'addons_path', ",".join([ + str(path) for path in valid_paths + ])) flush_streams() -def setup_server_wide_modules(config): +def setup_server_wide_modules(config, valid_paths): print("Searching for server wide modules") - all_manifests = get_all_manifests() + filters = set(['installable']) + all_manifests = find_modules_paths(valid_paths, filters) - print("Found %s modules" % (len(all_manifests))) + print("Found %s installable modules " % (len(all_manifests),)) - manifests = { - filename: get_module(filename) - for filename in all_manifests - } - - installable_manifests = { - filename: module - for filename, module in manifests.items() - if is_installable(module) - } - - print("Found %s installable modules " % (len(installable_manifests),)) - - server_wide_modules = get_server_wide_modules(installable_manifests) + server_wide_modules = get_server_wide_modules(all_manifests) if len(server_wide_modules) > 2: modules = ",".join(server_wide_modules) @@ -408,6 +290,36 @@ def setup_server_wide_modules(config): flush_streams() +def convert_value(name, value): + if value in ['True', 'False']: + return value == 'True' + + return value + + +def setup_env_config(config): + try: + from odoo.tools import config as odoo_config + except ImportError: + from openerp.tools import config as odoo_config + + params_by_name = {} + for key, opt in odoo_config.casts.items(): + value_opt = opt.get_opt_string() + value_opt = value_opt.upper().replace('--', 'ODOO_') + value_opt = value_opt.replace('-', '_') + params_by_name[value_opt] = key + + for key, value in os.environ.items(): + if not key.startswith('ODOO_'): + continue + + if key in params_by_name: + config_name = params_by_name[key] + converted_value = convert_value(key, value) + config.set('options', config_name, converted_value) + + def setup_environ(config): print("Configuring environment variables for postgresql") @@ -499,12 +411,39 @@ def wait_postgresql(): @contextmanager def get_config(config_path): config = ConfigParser() - config.read(config_path) + config.read(str(config_path)) yield config - with config_path.open('w') as out: - config.write(out) + try: + with config_path.open('w') as out: + config.write(out) + except Exception: + with config_path.open('wb') as out: + config.write(out) + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for vpath in paths: + cur_path = Path(vpath) + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(vpath) + + return res_paths def main(): @@ -518,7 +457,9 @@ def main(): sys.exit(ret) # Install python packages with pip in user home - valid_paths = get_valid_paths() + orig_valid_paths = get_valid_paths() + excluded_paths = get_excluded_paths() + valid_paths = filter_valid_paths(orig_valid_paths, excluded_paths) config_path = Path(os.environ.get('ODOO_RC', '/etc/odoo/odoo.cfg')) @@ -528,7 +469,8 @@ def main(): install_master_password(config) setup_environ(config) setup_addons_paths(config, valid_paths) - setup_server_wide_modules(config) + setup_server_wide_modules(config, valid_paths) + setup_env_config(config) if not os.environ.get('ODOO_SKIP_POSTGRES_WAIT'): wait_postgresql() diff --git a/10.0/find_modules.py b/10.0/find_modules.py deleted file mode 100644 index 9cb943748..000000000 --- a/10.0/find_modules.py +++ /dev/null @@ -1,76 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -from itertools import chain - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - "-p", - dest='paths', - action="append", - help="Location in which to search", - default=[] - ) - - parser.add_option( - "--only-name", - dest="only_name", - action="store_true", - help="Only display module name instead of path", - default=False, - ) - - parser.add_option( - '--csv', - dest="is_csv", - action="store_true", - help="Output as a comma separated list", - default=False - ) - - return parser - - -def find_modules(options, path): - modules = set() - - path = Path.cwd() / path - - erp_manifest = '__openerp__.py' - odoo_manifest = '__manifest__.py' - - manifest_globs = chain( - path.glob('**/{}'.format(erp_manifest)), - path.glob('**/{}'.format(odoo_manifest)), - ) - - for path in manifest_globs: - rel_path = path.parent.relative_to(Path.cwd()) - if options.only_name: - modules.add(rel_path.name) - else: - modules.add(str(rel_path)) - - return modules - - -def main(options, args): - modules = set() - for path in options.paths: - modules = modules.union(find_modules(options, path)) - - return modules - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - modules = main(options, args) - - if not options.is_csv: - for module in modules: - print(module) - else: - print(",".join(modules), end="") diff --git a/10.0/platform.sh b/10.0/platform.sh new file mode 100644 index 000000000..fdf5e42eb --- /dev/null +++ b/10.0/platform.sh @@ -0,0 +1,13 @@ +PLATFORM=`uname -m` + +if [ $PLATFORM = "aarch64" ] +then + echo -n "arm64" +else +if [ $PLATFORM = "x86_64" ] +then + echo -n "amd64" +else + echo -n $PLATFORM +fi +fi diff --git a/10.0/prepare_project.py b/10.0/prepare_project.py deleted file mode 100644 index 74a53f2e0..000000000 --- a/10.0/prepare_project.py +++ /dev/null @@ -1,198 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -import toml -from giturlparse import parse -from contextlib import contextmanager -import os -from subprocess import run -from urllib.parse import urlparse - - -def get_services(services): - return { - service.get('name'): service - for service in services.get('services') - } - - -def addons_by_project(options, addons): - res = {} - - for addon in addons: - if addon['url'] == 'self': - if options.ignore_self: - continue - parsed = parse(options.url) - else: - parsed = parse(addon['url']) - - auth = parsed.protocol in ['git', 'ssh'] - - res[parsed.repo] = dict( - addon, - url=parsed.url2https, - auth=auth - ) - - return res - - -def merge_addons(options, base, other): - base_addons = addons_by_project(options, base) - other_addons = addons_by_project(options, other) - - for name, addon in other_addons.items(): - if name not in base_addons: - base_addons[name] = addon - else: - base_addons[name] = dict(base_addons[name], **addon) - - return [ - addon - for addon in base_addons.values() - ] - - -def merge_services(options, base, other): - basic_inherit = dict(base, **other) - - if base.get('addons') or other.get('addons'): - basic_inherit['addons'] = merge_addons( - options, - base.get('addons', []), - other.get('addons', []) - ) - - return basic_inherit - - -def compile_service(options, services, name): - service = services.get(name, {}) - if 'inherit' in service: - merge_service = compile_service(options, services, service['inherit']) - service = merge_services(options, merge_service, service) - - return service - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - '-f', - '--file', - dest="file", - help="Input File" - ) - - parser.add_option( - '--url', - dest='url', - help="Url of self project" - ) - - parser.add_option( - '-o', - '--output', - dest="output_directory", - help="Output Directory" - ) - - parser.add_option( - '-e', - dest="env", - help="Environment to prepare" - ) - - parser.add_option( - '--username', - dest="username", - help="Username to replace with", - ) - - parser.add_option( - '--password', - dest="password", - help="password to set on https urls" - ) - - parser.add_option( - '-b', - '--branch', - dest="branch", - help="Default branch if no ref is defined" - ) - - parser.add_option( - '--ignore-self', - dest="ignore_self", - action="store_true", - help="Ignore self url as it's already fetched", - default=False - ) - - return parser - - -@contextmanager -def cd(directory): - cwd = Path.cwd() - try: - os.chdir(directory) - yield - finally: - os.chdir(cwd) - - -def fetch_addons(options, addon): - parsed = parse(addon['url']) - - url = urlparse(parsed.url2https) - - if addon['auth'] and options.username and options.password: - url = url._replace( - netloc="{}:{}@{}".format( - options.username, - options.password, - url.netloc - ) - ) - - repo_path = Path.cwd() / options.output_directory / parsed.repo - - repo_path.mkdir(exist_ok=True) - - with cd(repo_path): - run(['git', 'init']) - run(['git', 'remote', 'add', 'origin', url.geturl()]) - - ref = addon.get('commit') or addon.get('branch') or options.branch - - if ref: - run(['git', 'fetch', 'origin', ref]) - else: - run(['git', 'fetch', 'origin']) - - run(['git', 'checkout', 'FETCH_HEAD']) - run(['git', 'remote', 'remove', 'origin']) - - -def main(options, args): - with Path(options.file).open('r') as fin: - services = toml.loads(fin.read()) - - by_name = get_services(services) - - outputdir = Path(options.output_directory) - outputdir.mkdir(exist_ok=True) - - service = compile_service(options, by_name, options.env) - - for addons in service.get('addons', []): - fetch_addons(options, addons) - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - main(options, args) diff --git a/10.0/sudo-entrypoint.py b/10.0/sudo-entrypoint.py index 98779baae..916239170 100755 --- a/10.0/sudo-entrypoint.py +++ b/10.0/sudo-entrypoint.py @@ -12,6 +12,7 @@ from os import path from os.path import expanduser import shutil +import six try: from pathlib import Path @@ -69,6 +70,37 @@ def get_addons_paths(): ] +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for cur_path in paths: + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(cur_path) + + return res_paths + + def install_apt_packages(): """ Install debian dependencies. @@ -76,26 +108,34 @@ def install_apt_packages(): package_list = set() paths = get_addons_paths() + excluded_paths = get_excluded_paths() + paths = filter_valid_paths(paths, excluded_paths) print("Looking up for packages in {}".format(paths)) for addons_path in paths: for packages in addons_path.glob('**/apt-packages.txt'): print("Installing packages from %s" % packages) - with open(packages, 'r') as pack_file: - lines = [line.strip() for line in pack_file] + with packages.open('r') as pack_file: + lines = [ + six.ensure_text(line).strip() + for line in pack_file + if six.ensure_text(line).strip() + ] package_list.update(set(lines)) extras = os.environ.get('EXTRA_APT_PACKAGES', '') - print(f"Adding extra packages {extras}") + print("Adding extra packages {extras}".format(extras=extras)) if extras: for package in extras.split(','): - if not package: + if not package.strip(): continue - package_list.add(package) + package_list.add( + six.ensure_text(package).strip() + ) if len(package_list) > 0: - print(f"Installing {package_list}") + print("Installing {package_list}".format(package_list=package_list)) ret = pipe(['apt-get', 'update']) # Something went wrong, stop the service as it's failing diff --git a/11.0-nightly/Dockerfile b/11.0-nightly/Dockerfile index bc58326e7..39065b71a 100644 --- a/11.0-nightly/Dockerfile +++ b/11.0-nightly/Dockerfile @@ -5,89 +5,93 @@ LABEL maintainer="Archeti " ENV LANG C.UTF-8 ARG DEBIAN_FRONTEND=noninteractive +COPY ./platform.sh /platform.sh # Install some dependencies python3.7 RUN set -x; \ - apt-get update \ + export ARCH=`sh /platform.sh` \ + # Setup deb source for postgresql client + && apt-get update \ && apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg \ + python3.7 \ python3-wheel \ python3-setuptools \ python3-pip \ - # python3.7 \ - # libpython3.7 \ + python3-psycopg2 \ + python3-ldap \ + python3-libsass \ + python3-lxml \ + python3-pillow \ + python3-pypdf2 \ + python3-psutil \ + python3-asn1crypto \ curl \ - gnupg \ - libpq-dev \ - libsasl2-2 \ - libldap-2.4-2 \ - libxml2 \ - libxmlsec1 \ - libxslt1.1 \ + # libpq-dev \ + # libsasl2-2 \ + # libldap-2.4-2 \ + # libxml2 \ + # libxmlsec1 \ + # libxslt1.1 \ sudo \ node-less \ - # python3-yaml \ - && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 \ - # && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 2 \ - && update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1 \ + && echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ + && export GNUPGHOME="$(mktemp -d)" \ + && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ + && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ + && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ + && gpgconf --kill all \ + && rm -rf "$GNUPGHOME" \ + # Download deb for wkhtmltox + && echo "Downloading https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb" \ + && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + ./wkhtmltox.deb \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 \ && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 2 \ - && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb \ - && apt-get install -y --no-install-recommends ./wkhtmltox.deb \ && rm -rf /var/lib/apt/lists/* wkhtmltox.deb -# Install latest postgresql-client -RUN set -x; \ - echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ - && export GNUPGHOME="$(mktemp -d)" \ - && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ - && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ - && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ - && gpgconf --kill all \ - && rm -rf "$GNUPGHOME" \ - && apt-get update \ - && apt-get install -y postgresql-client \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH=/usr/local/bin:$PATH +ENV PATH=/var/lib/odoo/.local/bin:/usr/local/bin:$PATH # Install Odoo Including things from sources ENV ODOO_VERSION 11.0 -ENV ODOO_RELEASE=20210625 +ENV ODOO_RELEASE= ARG ODOO_ARCHIVE=odoo_${ODOO_VERSION}.${ODOO_RELEASE}.tar.gz RUN set -x; \ apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev ruby-sass \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ + # build-essential \ + # python3.7-dev \ + # libsasl2-dev \ + # libldap2-dev ruby-sass \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ && pip install -U pip \ && /usr/bin/env pip install \ - psycogreen \ + odoo-tools \ \ - && curl -o requirements.txt https://raw.githubusercontent.com/odoo/odoo/${ODOO_VERSION}/requirements.txt \ - && pip install -r requirements.txt \ - && /usr/bin/env pip install https://nightly.odoo.com/${ODOO_VERSION}/nightly/src/${ODOO_ARCHIVE} \ + && odootools manage setup --release "" --repo "https://github.com/odoo/odoo.git" --ref "" "11.0" \ && cd / \ && apt-get --purge remove -y \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ + # build-essential \ + # python3.7-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* - -VOLUME /etc/odoo -VOLUME /var/lib/odoo + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /root/.cache -COPY ./odoo.conf /etc/odoo/ -COPY ./entrypoint.py / -COPY ./sudo-entrypoint.py / -COPY ./find_modules.py /scripts/find_modules.py -COPY ./prepare_project.py /scripts/prepare_project.py +# COPY ./odoo.conf /etc/odoo/ +# COPY ./entrypoint.py / +# COPY ./sudo-entrypoint.py / ARG UID=1000 ARG GID=1000 @@ -95,12 +99,15 @@ ARG GID=1000 RUN mkdir /addons \ && groupadd -r -g ${GID} odoo \ && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ - && chown odoo /etc/odoo/odoo.conf \ + # && chown odoo /etc/odoo/odoo.conf \ && chown -R odoo:odoo /addons \ - && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /sudo-entrypoint.py" >> /etc/sudoers + && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /usr/local/bin/odootools" >> /etc/sudoers + +# VOLUME /etc/odoo +VOLUME /var/lib/odoo -ENV ODOO_RC /etc/odoo/odoo.conf -ENV ODOO_BASE_PATH /usr/local/lib/python3.6/dist-packages/odoo/addons +ENV ODOO_RC /var/lib/odoo/odoo.conf +ENV ODOO_BASE_PATH /usr/local/lib/python3.7/dist-packages/odoo/addons # Env variable defined to monitor the kind of service running # it could be a staging/production/test or anything and undefined # is the default in case we need to know servers that aren't correctly @@ -113,19 +120,19 @@ expose 8071 USER odoo LABEL version="11.0" -LABEL release="20210625" +LABEL release="" -LABEL org.opencontainers.image.created="2021-06-25T17:55:41.199454" +LABEL org.opencontainers.image.created="2022-11-04T20:06:13.822066" LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" -LABEL org.opencontainers.image.authors="Archeti " +LABEL org.opencontainers.image.authors="OdooPlus " LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.version="11.0" -LABEL org.opencontainers.image.vendor="ArcheTI" +LABEL org.opencontainers.image.vendor="OdooPlus" LABEL org.opencontainers.image.ref.name="11.0-nightly" LABEL org.opencontainers.image.title="Odoo 11.0" LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." -ENTRYPOINT ["/entrypoint.py"] +ENTRYPOINT ["odootools", "--log-level", "INFO", "entrypoint", "user"] cmd ["odoo"] diff --git a/11.0-nightly/entrypoint.py b/11.0-nightly/entrypoint.py index f2ce4654b..bb0ffddc3 100755 --- a/11.0-nightly/entrypoint.py +++ b/11.0-nightly/entrypoint.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # import argparse +from __future__ import print_function import time import shlex import subprocess @@ -17,25 +18,9 @@ from passlib.context import CryptContext from contextlib import contextmanager -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - -try: - from pip._internal.network.session import PipSession - from pip._internal.req.req_file import parse_requirements - from pip._internal.req.constructors import ( - install_req_from_parsed_requirement - ) -except Exception: - from pip.download import PipSession - from pip.req.req_file import parse_requirements - - def install_req_from_parsed_requirement(req): - return req - -from collections import defaultdict +from odoo_tools.compat import Path +from odoo_tools.modules.search import find_addons_paths, find_modules_paths +from odoo_tools.configuration.pips import merge_requirements try: # python3 @@ -68,63 +53,6 @@ def quote(s): return "'" + s.replace("'", "'\"'\"'") + "'" -class Requirement(object): - def __init__(self): - self.extras = set() - self.specifiers = set() - self.links = set() - self.editable = False - - -def merge_requirements(files): - requirements = defaultdict(lambda: Requirement()) - links = set() - - for filename in files: - f_requirements = parse_requirements( - str(filename), session=PipSession() - ) - for parsed_requirement in f_requirements: - requirement = install_req_from_parsed_requirement( - parsed_requirement - ) - - if requirement.markers and not requirement.markers.evaluate(): - continue - - if not hasattr(requirement.req, 'name'): - links.add(requirement.link.url) - break - name = requirement.req.name - specifiers = requirement.req.specifier - extras = requirement.req.extras - requirements[name].extras |= set(extras) - requirements[name].specifiers |= set(specifiers) - if requirement.link: - requirements[name].links |= {requirement.link.url} - requirements[name].editable |= requirement.editable - - result = [] - for key, value in requirements.items(): - if value.links: - result.append("%s" % value.links.pop()) - elif not value.extras: - result.append( - "%s %s" % (key, ",".join(map(str, value.specifiers))) - ) - else: - result.append("%s [%s] %s" % ( - key, - ",".join(map(str, value.extras)), - ",".join(map(str, value.specifiers)) - )) - - for link in links: - result.append(link) - - return "\n".join(result) - - def pipe(args): """ Call the process with std(in,out,err) @@ -178,42 +106,12 @@ def call_sudo_entrypoint(): return ret -def get_all_manifests(): - odoo_manifests = glob.glob('/addons/**/__manifest__.py', recursive=True) - erp_manifest = glob.glob('/addons/**/__openerp__.py', recursive=True) - - return odoo_manifests + erp_manifest - - -def module_name(manifest_file): - return path.basename(path.dirname(manifest_file)) - - -def get_module(manifest_file): - with open(manifest_file) as manifest_in: - try: - code = compile(manifest_in.read(), '__manifest__', 'eval') - module = eval(code, {}, {}) - except Exception: - module = {} - - return module - - -def is_installable(manifest): - return manifest.get('installable', True) - - -def is_server_wide(manifest): - return manifest.get('server_wide', False) - - def get_server_wide_modules(manifests): base_server_wide_modules = ['base', 'web'] custom_server_wide_modules = [ - module_name(filename) - for filename, module in manifests.items() - if is_server_wide(module) + manifest.technical_name + for manifest in manifests + if manifest.server_wide ] return base_server_wide_modules + custom_server_wide_modules @@ -246,7 +144,12 @@ def install_python_dependencies(valid_paths): req_file = Path('/var/lib/odoo/requirements.txt') with req_file.open('w') as fout: data = merge_requirements(requirement_files) - fout.write(data) + data = "\n".join(data) + + try: + fout.write(data) + except Exception: + fout.write(data.decode('utf-8')) for requirements in requirement_files: print("Installing python packages from %s" % requirements) @@ -338,65 +241,44 @@ def get_extra_paths(): ] -def get_valid_paths(): - base_addons = os.environ.get('ODOO_BASE_PATH') +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] - # addons = os.listdir('/addons') - valid_paths = [base_addons] +def get_valid_paths(): + base_addons = os.environ['ODOO_BASE_PATH'] - addons_paths = get_dirs('/addons') - addons_paths += get_extra_paths() + base_addons_paths = [base_addons, '/addons'] + base_addons_paths += get_extra_paths() - for addons_path in addons_paths: - print("Lookup addons in %s" % addons_path) - flush_streams() - addons = get_dirs(addons_path) - for addon in addons: - files = os.listdir(addon) - if ( - '__init__.py' in files and - '__manifest__.py' in files or - '__openerp__.py' in files - ): - valid_paths.append(addons_path) - print("Addons found !") - flush_streams() - break - if addons_path in valid_paths: - continue - else: - print("No addons found in path. Skipping...") - flush_streams() + valid_paths = find_addons_paths(base_addons_paths) + + for valid_path in valid_paths: + print("Found installable modules in {}".format(valid_path)) return valid_paths def setup_addons_paths(config, valid_paths): - config.set('options', 'addons_path', ",".join(valid_paths)) + config.set('options', 'addons_path', ",".join([ + str(path) for path in valid_paths + ])) flush_streams() -def setup_server_wide_modules(config): +def setup_server_wide_modules(config, valid_paths): print("Searching for server wide modules") - all_manifests = get_all_manifests() + filters = set(['installable']) + all_manifests = find_modules_paths(valid_paths, filters) - print("Found %s modules" % (len(all_manifests))) + print("Found %s installable modules " % (len(all_manifests),)) - manifests = { - filename: get_module(filename) - for filename in all_manifests - } - - installable_manifests = { - filename: module - for filename, module in manifests.items() - if is_installable(module) - } - - print("Found %s installable modules " % (len(installable_manifests),)) - - server_wide_modules = get_server_wide_modules(installable_manifests) + server_wide_modules = get_server_wide_modules(all_manifests) if len(server_wide_modules) > 2: modules = ",".join(server_wide_modules) @@ -408,6 +290,36 @@ def setup_server_wide_modules(config): flush_streams() +def convert_value(name, value): + if value in ['True', 'False']: + return value == 'True' + + return value + + +def setup_env_config(config): + try: + from odoo.tools import config as odoo_config + except ImportError: + from openerp.tools import config as odoo_config + + params_by_name = {} + for key, opt in odoo_config.casts.items(): + value_opt = opt.get_opt_string() + value_opt = value_opt.upper().replace('--', 'ODOO_') + value_opt = value_opt.replace('-', '_') + params_by_name[value_opt] = key + + for key, value in os.environ.items(): + if not key.startswith('ODOO_'): + continue + + if key in params_by_name: + config_name = params_by_name[key] + converted_value = convert_value(key, value) + config.set('options', config_name, converted_value) + + def setup_environ(config): print("Configuring environment variables for postgresql") @@ -499,12 +411,39 @@ def wait_postgresql(): @contextmanager def get_config(config_path): config = ConfigParser() - config.read(config_path) + config.read(str(config_path)) yield config - with config_path.open('w') as out: - config.write(out) + try: + with config_path.open('w') as out: + config.write(out) + except Exception: + with config_path.open('wb') as out: + config.write(out) + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for vpath in paths: + cur_path = Path(vpath) + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(vpath) + + return res_paths def main(): @@ -518,7 +457,9 @@ def main(): sys.exit(ret) # Install python packages with pip in user home - valid_paths = get_valid_paths() + orig_valid_paths = get_valid_paths() + excluded_paths = get_excluded_paths() + valid_paths = filter_valid_paths(orig_valid_paths, excluded_paths) config_path = Path(os.environ.get('ODOO_RC', '/etc/odoo/odoo.cfg')) @@ -528,7 +469,8 @@ def main(): install_master_password(config) setup_environ(config) setup_addons_paths(config, valid_paths) - setup_server_wide_modules(config) + setup_server_wide_modules(config, valid_paths) + setup_env_config(config) if not os.environ.get('ODOO_SKIP_POSTGRES_WAIT'): wait_postgresql() diff --git a/11.0-nightly/find_modules.py b/11.0-nightly/find_modules.py deleted file mode 100644 index 9cb943748..000000000 --- a/11.0-nightly/find_modules.py +++ /dev/null @@ -1,76 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -from itertools import chain - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - "-p", - dest='paths', - action="append", - help="Location in which to search", - default=[] - ) - - parser.add_option( - "--only-name", - dest="only_name", - action="store_true", - help="Only display module name instead of path", - default=False, - ) - - parser.add_option( - '--csv', - dest="is_csv", - action="store_true", - help="Output as a comma separated list", - default=False - ) - - return parser - - -def find_modules(options, path): - modules = set() - - path = Path.cwd() / path - - erp_manifest = '__openerp__.py' - odoo_manifest = '__manifest__.py' - - manifest_globs = chain( - path.glob('**/{}'.format(erp_manifest)), - path.glob('**/{}'.format(odoo_manifest)), - ) - - for path in manifest_globs: - rel_path = path.parent.relative_to(Path.cwd()) - if options.only_name: - modules.add(rel_path.name) - else: - modules.add(str(rel_path)) - - return modules - - -def main(options, args): - modules = set() - for path in options.paths: - modules = modules.union(find_modules(options, path)) - - return modules - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - modules = main(options, args) - - if not options.is_csv: - for module in modules: - print(module) - else: - print(",".join(modules), end="") diff --git a/11.0-nightly/platform.sh b/11.0-nightly/platform.sh new file mode 100644 index 000000000..fdf5e42eb --- /dev/null +++ b/11.0-nightly/platform.sh @@ -0,0 +1,13 @@ +PLATFORM=`uname -m` + +if [ $PLATFORM = "aarch64" ] +then + echo -n "arm64" +else +if [ $PLATFORM = "x86_64" ] +then + echo -n "amd64" +else + echo -n $PLATFORM +fi +fi diff --git a/11.0-nightly/prepare_project.py b/11.0-nightly/prepare_project.py deleted file mode 100644 index 74a53f2e0..000000000 --- a/11.0-nightly/prepare_project.py +++ /dev/null @@ -1,198 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -import toml -from giturlparse import parse -from contextlib import contextmanager -import os -from subprocess import run -from urllib.parse import urlparse - - -def get_services(services): - return { - service.get('name'): service - for service in services.get('services') - } - - -def addons_by_project(options, addons): - res = {} - - for addon in addons: - if addon['url'] == 'self': - if options.ignore_self: - continue - parsed = parse(options.url) - else: - parsed = parse(addon['url']) - - auth = parsed.protocol in ['git', 'ssh'] - - res[parsed.repo] = dict( - addon, - url=parsed.url2https, - auth=auth - ) - - return res - - -def merge_addons(options, base, other): - base_addons = addons_by_project(options, base) - other_addons = addons_by_project(options, other) - - for name, addon in other_addons.items(): - if name not in base_addons: - base_addons[name] = addon - else: - base_addons[name] = dict(base_addons[name], **addon) - - return [ - addon - for addon in base_addons.values() - ] - - -def merge_services(options, base, other): - basic_inherit = dict(base, **other) - - if base.get('addons') or other.get('addons'): - basic_inherit['addons'] = merge_addons( - options, - base.get('addons', []), - other.get('addons', []) - ) - - return basic_inherit - - -def compile_service(options, services, name): - service = services.get(name, {}) - if 'inherit' in service: - merge_service = compile_service(options, services, service['inherit']) - service = merge_services(options, merge_service, service) - - return service - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - '-f', - '--file', - dest="file", - help="Input File" - ) - - parser.add_option( - '--url', - dest='url', - help="Url of self project" - ) - - parser.add_option( - '-o', - '--output', - dest="output_directory", - help="Output Directory" - ) - - parser.add_option( - '-e', - dest="env", - help="Environment to prepare" - ) - - parser.add_option( - '--username', - dest="username", - help="Username to replace with", - ) - - parser.add_option( - '--password', - dest="password", - help="password to set on https urls" - ) - - parser.add_option( - '-b', - '--branch', - dest="branch", - help="Default branch if no ref is defined" - ) - - parser.add_option( - '--ignore-self', - dest="ignore_self", - action="store_true", - help="Ignore self url as it's already fetched", - default=False - ) - - return parser - - -@contextmanager -def cd(directory): - cwd = Path.cwd() - try: - os.chdir(directory) - yield - finally: - os.chdir(cwd) - - -def fetch_addons(options, addon): - parsed = parse(addon['url']) - - url = urlparse(parsed.url2https) - - if addon['auth'] and options.username and options.password: - url = url._replace( - netloc="{}:{}@{}".format( - options.username, - options.password, - url.netloc - ) - ) - - repo_path = Path.cwd() / options.output_directory / parsed.repo - - repo_path.mkdir(exist_ok=True) - - with cd(repo_path): - run(['git', 'init']) - run(['git', 'remote', 'add', 'origin', url.geturl()]) - - ref = addon.get('commit') or addon.get('branch') or options.branch - - if ref: - run(['git', 'fetch', 'origin', ref]) - else: - run(['git', 'fetch', 'origin']) - - run(['git', 'checkout', 'FETCH_HEAD']) - run(['git', 'remote', 'remove', 'origin']) - - -def main(options, args): - with Path(options.file).open('r') as fin: - services = toml.loads(fin.read()) - - by_name = get_services(services) - - outputdir = Path(options.output_directory) - outputdir.mkdir(exist_ok=True) - - service = compile_service(options, by_name, options.env) - - for addons in service.get('addons', []): - fetch_addons(options, addons) - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - main(options, args) diff --git a/11.0-nightly/sudo-entrypoint.py b/11.0-nightly/sudo-entrypoint.py index 98779baae..916239170 100755 --- a/11.0-nightly/sudo-entrypoint.py +++ b/11.0-nightly/sudo-entrypoint.py @@ -12,6 +12,7 @@ from os import path from os.path import expanduser import shutil +import six try: from pathlib import Path @@ -69,6 +70,37 @@ def get_addons_paths(): ] +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for cur_path in paths: + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(cur_path) + + return res_paths + + def install_apt_packages(): """ Install debian dependencies. @@ -76,26 +108,34 @@ def install_apt_packages(): package_list = set() paths = get_addons_paths() + excluded_paths = get_excluded_paths() + paths = filter_valid_paths(paths, excluded_paths) print("Looking up for packages in {}".format(paths)) for addons_path in paths: for packages in addons_path.glob('**/apt-packages.txt'): print("Installing packages from %s" % packages) - with open(packages, 'r') as pack_file: - lines = [line.strip() for line in pack_file] + with packages.open('r') as pack_file: + lines = [ + six.ensure_text(line).strip() + for line in pack_file + if six.ensure_text(line).strip() + ] package_list.update(set(lines)) extras = os.environ.get('EXTRA_APT_PACKAGES', '') - print(f"Adding extra packages {extras}") + print("Adding extra packages {extras}".format(extras=extras)) if extras: for package in extras.split(','): - if not package: + if not package.strip(): continue - package_list.add(package) + package_list.add( + six.ensure_text(package).strip() + ) if len(package_list) > 0: - print(f"Installing {package_list}") + print("Installing {package_list}".format(package_list=package_list)) ret = pipe(['apt-get', 'update']) # Something went wrong, stop the service as it's failing diff --git a/11.0/Dockerfile b/11.0/Dockerfile index 309f11ad8..75b471609 100644 --- a/11.0/Dockerfile +++ b/11.0/Dockerfile @@ -5,89 +5,93 @@ LABEL maintainer="Archeti " ENV LANG C.UTF-8 ARG DEBIAN_FRONTEND=noninteractive +COPY ./platform.sh /platform.sh # Install some dependencies python3.7 RUN set -x; \ - apt-get update \ + export ARCH=`sh /platform.sh` \ + # Setup deb source for postgresql client + && apt-get update \ && apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg \ + python3.6 \ python3-wheel \ python3-setuptools \ python3-pip \ - # python3.7 \ - # libpython3.7 \ + python3-psycopg2 \ + python3-ldap \ + python3-libsass \ + python3-lxml \ + python3-pillow \ + python3-pypdf2 \ + python3-psutil \ + python3-asn1crypto \ curl \ - gnupg \ - libpq-dev \ - libsasl2-2 \ - libldap-2.4-2 \ - libxml2 \ - libxmlsec1 \ - libxslt1.1 \ + # libpq-dev \ + # libsasl2-2 \ + # libldap-2.4-2 \ + # libxml2 \ + # libxmlsec1 \ + # libxslt1.1 \ sudo \ node-less \ - # python3-yaml \ + && echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ + && export GNUPGHOME="$(mktemp -d)" \ + && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ + && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ + && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ + && gpgconf --kill all \ + && rm -rf "$GNUPGHOME" \ + # Download deb for wkhtmltox + && echo "Downloading https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb" \ + && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + ./wkhtmltox.deb \ && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 \ - # && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 2 \ && update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1 \ && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 2 \ - && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb \ - && apt-get install -y --no-install-recommends ./wkhtmltox.deb \ && rm -rf /var/lib/apt/lists/* wkhtmltox.deb -# Install latest postgresql-client -RUN set -x; \ - echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ - && export GNUPGHOME="$(mktemp -d)" \ - && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ - && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ - && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ - && gpgconf --kill all \ - && rm -rf "$GNUPGHOME" \ - && apt-get update \ - && apt-get install -y postgresql-client \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH=/usr/local/bin:$PATH +ENV PATH=/var/lib/odoo/.local/bin:/usr/local/bin:$PATH # Install Odoo Including things from sources ENV ODOO_VERSION 11.0 -ENV ODOO_RELEASE=20200623 +ENV ODOO_RELEASE=20210922 ARG ODOO_ARCHIVE=odoo_${ODOO_VERSION}.${ODOO_RELEASE}.tar.gz RUN set -x; \ apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev ruby-sass \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ + # build-essential \ + # python3.6-dev \ + # libsasl2-dev \ + # libldap2-dev ruby-sass \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ && pip install -U pip \ && /usr/bin/env pip install \ - psycogreen \ + odoo-tools \ \ - && curl -o requirements.txt https://raw.githubusercontent.com/odoo/odoo/${ODOO_VERSION}/requirements.txt \ - && pip install -r requirements.txt \ - && /usr/bin/env pip install https://nightly.odoo.com/${ODOO_VERSION}/nightly/src/${ODOO_ARCHIVE} \ + && odootools manage setup --release "20210922" --repo "https://github.com/odoo/odoo.git" --ref "" "11.0" \ && cd / \ && apt-get --purge remove -y \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ + # build-essential \ + # python3.6-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* - -VOLUME /etc/odoo -VOLUME /var/lib/odoo + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /root/.cache -COPY ./odoo.conf /etc/odoo/ -COPY ./entrypoint.py / -COPY ./sudo-entrypoint.py / -COPY ./find_modules.py /scripts/find_modules.py -COPY ./prepare_project.py /scripts/prepare_project.py +# COPY ./odoo.conf /etc/odoo/ +# COPY ./entrypoint.py / +# COPY ./sudo-entrypoint.py / ARG UID=1000 ARG GID=1000 @@ -95,11 +99,14 @@ ARG GID=1000 RUN mkdir /addons \ && groupadd -r -g ${GID} odoo \ && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ - && chown odoo /etc/odoo/odoo.conf \ + # && chown odoo /etc/odoo/odoo.conf \ && chown -R odoo:odoo /addons \ - && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /sudo-entrypoint.py" >> /etc/sudoers + && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /usr/local/bin/odootools" >> /etc/sudoers + +# VOLUME /etc/odoo +VOLUME /var/lib/odoo -ENV ODOO_RC /etc/odoo/odoo.conf +ENV ODOO_RC /var/lib/odoo/odoo.conf ENV ODOO_BASE_PATH /usr/local/lib/python3.6/dist-packages/odoo/addons # Env variable defined to monitor the kind of service running # it could be a staging/production/test or anything and undefined @@ -113,19 +120,19 @@ expose 8071 USER odoo LABEL version="11.0" -LABEL release="20200623" +LABEL release="20210922" -LABEL org.opencontainers.image.created="2021-06-25T17:55:41.194840" +LABEL org.opencontainers.image.created="2022-11-04T20:06:13.800893" LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" -LABEL org.opencontainers.image.authors="Archeti " +LABEL org.opencontainers.image.authors="OdooPlus " LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.version="11.0" -LABEL org.opencontainers.image.vendor="ArcheTI" +LABEL org.opencontainers.image.vendor="OdooPlus" LABEL org.opencontainers.image.ref.name="11.0" LABEL org.opencontainers.image.title="Odoo 11.0" LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." -ENTRYPOINT ["/entrypoint.py"] +ENTRYPOINT ["odootools", "--log-level", "INFO", "entrypoint", "user"] cmd ["odoo"] diff --git a/11.0/entrypoint.py b/11.0/entrypoint.py index f2ce4654b..bb0ffddc3 100755 --- a/11.0/entrypoint.py +++ b/11.0/entrypoint.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # import argparse +from __future__ import print_function import time import shlex import subprocess @@ -17,25 +18,9 @@ from passlib.context import CryptContext from contextlib import contextmanager -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - -try: - from pip._internal.network.session import PipSession - from pip._internal.req.req_file import parse_requirements - from pip._internal.req.constructors import ( - install_req_from_parsed_requirement - ) -except Exception: - from pip.download import PipSession - from pip.req.req_file import parse_requirements - - def install_req_from_parsed_requirement(req): - return req - -from collections import defaultdict +from odoo_tools.compat import Path +from odoo_tools.modules.search import find_addons_paths, find_modules_paths +from odoo_tools.configuration.pips import merge_requirements try: # python3 @@ -68,63 +53,6 @@ def quote(s): return "'" + s.replace("'", "'\"'\"'") + "'" -class Requirement(object): - def __init__(self): - self.extras = set() - self.specifiers = set() - self.links = set() - self.editable = False - - -def merge_requirements(files): - requirements = defaultdict(lambda: Requirement()) - links = set() - - for filename in files: - f_requirements = parse_requirements( - str(filename), session=PipSession() - ) - for parsed_requirement in f_requirements: - requirement = install_req_from_parsed_requirement( - parsed_requirement - ) - - if requirement.markers and not requirement.markers.evaluate(): - continue - - if not hasattr(requirement.req, 'name'): - links.add(requirement.link.url) - break - name = requirement.req.name - specifiers = requirement.req.specifier - extras = requirement.req.extras - requirements[name].extras |= set(extras) - requirements[name].specifiers |= set(specifiers) - if requirement.link: - requirements[name].links |= {requirement.link.url} - requirements[name].editable |= requirement.editable - - result = [] - for key, value in requirements.items(): - if value.links: - result.append("%s" % value.links.pop()) - elif not value.extras: - result.append( - "%s %s" % (key, ",".join(map(str, value.specifiers))) - ) - else: - result.append("%s [%s] %s" % ( - key, - ",".join(map(str, value.extras)), - ",".join(map(str, value.specifiers)) - )) - - for link in links: - result.append(link) - - return "\n".join(result) - - def pipe(args): """ Call the process with std(in,out,err) @@ -178,42 +106,12 @@ def call_sudo_entrypoint(): return ret -def get_all_manifests(): - odoo_manifests = glob.glob('/addons/**/__manifest__.py', recursive=True) - erp_manifest = glob.glob('/addons/**/__openerp__.py', recursive=True) - - return odoo_manifests + erp_manifest - - -def module_name(manifest_file): - return path.basename(path.dirname(manifest_file)) - - -def get_module(manifest_file): - with open(manifest_file) as manifest_in: - try: - code = compile(manifest_in.read(), '__manifest__', 'eval') - module = eval(code, {}, {}) - except Exception: - module = {} - - return module - - -def is_installable(manifest): - return manifest.get('installable', True) - - -def is_server_wide(manifest): - return manifest.get('server_wide', False) - - def get_server_wide_modules(manifests): base_server_wide_modules = ['base', 'web'] custom_server_wide_modules = [ - module_name(filename) - for filename, module in manifests.items() - if is_server_wide(module) + manifest.technical_name + for manifest in manifests + if manifest.server_wide ] return base_server_wide_modules + custom_server_wide_modules @@ -246,7 +144,12 @@ def install_python_dependencies(valid_paths): req_file = Path('/var/lib/odoo/requirements.txt') with req_file.open('w') as fout: data = merge_requirements(requirement_files) - fout.write(data) + data = "\n".join(data) + + try: + fout.write(data) + except Exception: + fout.write(data.decode('utf-8')) for requirements in requirement_files: print("Installing python packages from %s" % requirements) @@ -338,65 +241,44 @@ def get_extra_paths(): ] -def get_valid_paths(): - base_addons = os.environ.get('ODOO_BASE_PATH') +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] - # addons = os.listdir('/addons') - valid_paths = [base_addons] +def get_valid_paths(): + base_addons = os.environ['ODOO_BASE_PATH'] - addons_paths = get_dirs('/addons') - addons_paths += get_extra_paths() + base_addons_paths = [base_addons, '/addons'] + base_addons_paths += get_extra_paths() - for addons_path in addons_paths: - print("Lookup addons in %s" % addons_path) - flush_streams() - addons = get_dirs(addons_path) - for addon in addons: - files = os.listdir(addon) - if ( - '__init__.py' in files and - '__manifest__.py' in files or - '__openerp__.py' in files - ): - valid_paths.append(addons_path) - print("Addons found !") - flush_streams() - break - if addons_path in valid_paths: - continue - else: - print("No addons found in path. Skipping...") - flush_streams() + valid_paths = find_addons_paths(base_addons_paths) + + for valid_path in valid_paths: + print("Found installable modules in {}".format(valid_path)) return valid_paths def setup_addons_paths(config, valid_paths): - config.set('options', 'addons_path', ",".join(valid_paths)) + config.set('options', 'addons_path', ",".join([ + str(path) for path in valid_paths + ])) flush_streams() -def setup_server_wide_modules(config): +def setup_server_wide_modules(config, valid_paths): print("Searching for server wide modules") - all_manifests = get_all_manifests() + filters = set(['installable']) + all_manifests = find_modules_paths(valid_paths, filters) - print("Found %s modules" % (len(all_manifests))) + print("Found %s installable modules " % (len(all_manifests),)) - manifests = { - filename: get_module(filename) - for filename in all_manifests - } - - installable_manifests = { - filename: module - for filename, module in manifests.items() - if is_installable(module) - } - - print("Found %s installable modules " % (len(installable_manifests),)) - - server_wide_modules = get_server_wide_modules(installable_manifests) + server_wide_modules = get_server_wide_modules(all_manifests) if len(server_wide_modules) > 2: modules = ",".join(server_wide_modules) @@ -408,6 +290,36 @@ def setup_server_wide_modules(config): flush_streams() +def convert_value(name, value): + if value in ['True', 'False']: + return value == 'True' + + return value + + +def setup_env_config(config): + try: + from odoo.tools import config as odoo_config + except ImportError: + from openerp.tools import config as odoo_config + + params_by_name = {} + for key, opt in odoo_config.casts.items(): + value_opt = opt.get_opt_string() + value_opt = value_opt.upper().replace('--', 'ODOO_') + value_opt = value_opt.replace('-', '_') + params_by_name[value_opt] = key + + for key, value in os.environ.items(): + if not key.startswith('ODOO_'): + continue + + if key in params_by_name: + config_name = params_by_name[key] + converted_value = convert_value(key, value) + config.set('options', config_name, converted_value) + + def setup_environ(config): print("Configuring environment variables for postgresql") @@ -499,12 +411,39 @@ def wait_postgresql(): @contextmanager def get_config(config_path): config = ConfigParser() - config.read(config_path) + config.read(str(config_path)) yield config - with config_path.open('w') as out: - config.write(out) + try: + with config_path.open('w') as out: + config.write(out) + except Exception: + with config_path.open('wb') as out: + config.write(out) + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for vpath in paths: + cur_path = Path(vpath) + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(vpath) + + return res_paths def main(): @@ -518,7 +457,9 @@ def main(): sys.exit(ret) # Install python packages with pip in user home - valid_paths = get_valid_paths() + orig_valid_paths = get_valid_paths() + excluded_paths = get_excluded_paths() + valid_paths = filter_valid_paths(orig_valid_paths, excluded_paths) config_path = Path(os.environ.get('ODOO_RC', '/etc/odoo/odoo.cfg')) @@ -528,7 +469,8 @@ def main(): install_master_password(config) setup_environ(config) setup_addons_paths(config, valid_paths) - setup_server_wide_modules(config) + setup_server_wide_modules(config, valid_paths) + setup_env_config(config) if not os.environ.get('ODOO_SKIP_POSTGRES_WAIT'): wait_postgresql() diff --git a/11.0/find_modules.py b/11.0/find_modules.py deleted file mode 100644 index 9cb943748..000000000 --- a/11.0/find_modules.py +++ /dev/null @@ -1,76 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -from itertools import chain - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - "-p", - dest='paths', - action="append", - help="Location in which to search", - default=[] - ) - - parser.add_option( - "--only-name", - dest="only_name", - action="store_true", - help="Only display module name instead of path", - default=False, - ) - - parser.add_option( - '--csv', - dest="is_csv", - action="store_true", - help="Output as a comma separated list", - default=False - ) - - return parser - - -def find_modules(options, path): - modules = set() - - path = Path.cwd() / path - - erp_manifest = '__openerp__.py' - odoo_manifest = '__manifest__.py' - - manifest_globs = chain( - path.glob('**/{}'.format(erp_manifest)), - path.glob('**/{}'.format(odoo_manifest)), - ) - - for path in manifest_globs: - rel_path = path.parent.relative_to(Path.cwd()) - if options.only_name: - modules.add(rel_path.name) - else: - modules.add(str(rel_path)) - - return modules - - -def main(options, args): - modules = set() - for path in options.paths: - modules = modules.union(find_modules(options, path)) - - return modules - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - modules = main(options, args) - - if not options.is_csv: - for module in modules: - print(module) - else: - print(",".join(modules), end="") diff --git a/11.0/platform.sh b/11.0/platform.sh new file mode 100644 index 000000000..fdf5e42eb --- /dev/null +++ b/11.0/platform.sh @@ -0,0 +1,13 @@ +PLATFORM=`uname -m` + +if [ $PLATFORM = "aarch64" ] +then + echo -n "arm64" +else +if [ $PLATFORM = "x86_64" ] +then + echo -n "amd64" +else + echo -n $PLATFORM +fi +fi diff --git a/11.0/prepare_project.py b/11.0/prepare_project.py deleted file mode 100644 index 74a53f2e0..000000000 --- a/11.0/prepare_project.py +++ /dev/null @@ -1,198 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -import toml -from giturlparse import parse -from contextlib import contextmanager -import os -from subprocess import run -from urllib.parse import urlparse - - -def get_services(services): - return { - service.get('name'): service - for service in services.get('services') - } - - -def addons_by_project(options, addons): - res = {} - - for addon in addons: - if addon['url'] == 'self': - if options.ignore_self: - continue - parsed = parse(options.url) - else: - parsed = parse(addon['url']) - - auth = parsed.protocol in ['git', 'ssh'] - - res[parsed.repo] = dict( - addon, - url=parsed.url2https, - auth=auth - ) - - return res - - -def merge_addons(options, base, other): - base_addons = addons_by_project(options, base) - other_addons = addons_by_project(options, other) - - for name, addon in other_addons.items(): - if name not in base_addons: - base_addons[name] = addon - else: - base_addons[name] = dict(base_addons[name], **addon) - - return [ - addon - for addon in base_addons.values() - ] - - -def merge_services(options, base, other): - basic_inherit = dict(base, **other) - - if base.get('addons') or other.get('addons'): - basic_inherit['addons'] = merge_addons( - options, - base.get('addons', []), - other.get('addons', []) - ) - - return basic_inherit - - -def compile_service(options, services, name): - service = services.get(name, {}) - if 'inherit' in service: - merge_service = compile_service(options, services, service['inherit']) - service = merge_services(options, merge_service, service) - - return service - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - '-f', - '--file', - dest="file", - help="Input File" - ) - - parser.add_option( - '--url', - dest='url', - help="Url of self project" - ) - - parser.add_option( - '-o', - '--output', - dest="output_directory", - help="Output Directory" - ) - - parser.add_option( - '-e', - dest="env", - help="Environment to prepare" - ) - - parser.add_option( - '--username', - dest="username", - help="Username to replace with", - ) - - parser.add_option( - '--password', - dest="password", - help="password to set on https urls" - ) - - parser.add_option( - '-b', - '--branch', - dest="branch", - help="Default branch if no ref is defined" - ) - - parser.add_option( - '--ignore-self', - dest="ignore_self", - action="store_true", - help="Ignore self url as it's already fetched", - default=False - ) - - return parser - - -@contextmanager -def cd(directory): - cwd = Path.cwd() - try: - os.chdir(directory) - yield - finally: - os.chdir(cwd) - - -def fetch_addons(options, addon): - parsed = parse(addon['url']) - - url = urlparse(parsed.url2https) - - if addon['auth'] and options.username and options.password: - url = url._replace( - netloc="{}:{}@{}".format( - options.username, - options.password, - url.netloc - ) - ) - - repo_path = Path.cwd() / options.output_directory / parsed.repo - - repo_path.mkdir(exist_ok=True) - - with cd(repo_path): - run(['git', 'init']) - run(['git', 'remote', 'add', 'origin', url.geturl()]) - - ref = addon.get('commit') or addon.get('branch') or options.branch - - if ref: - run(['git', 'fetch', 'origin', ref]) - else: - run(['git', 'fetch', 'origin']) - - run(['git', 'checkout', 'FETCH_HEAD']) - run(['git', 'remote', 'remove', 'origin']) - - -def main(options, args): - with Path(options.file).open('r') as fin: - services = toml.loads(fin.read()) - - by_name = get_services(services) - - outputdir = Path(options.output_directory) - outputdir.mkdir(exist_ok=True) - - service = compile_service(options, by_name, options.env) - - for addons in service.get('addons', []): - fetch_addons(options, addons) - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - main(options, args) diff --git a/11.0/sudo-entrypoint.py b/11.0/sudo-entrypoint.py index 98779baae..916239170 100755 --- a/11.0/sudo-entrypoint.py +++ b/11.0/sudo-entrypoint.py @@ -12,6 +12,7 @@ from os import path from os.path import expanduser import shutil +import six try: from pathlib import Path @@ -69,6 +70,37 @@ def get_addons_paths(): ] +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for cur_path in paths: + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(cur_path) + + return res_paths + + def install_apt_packages(): """ Install debian dependencies. @@ -76,26 +108,34 @@ def install_apt_packages(): package_list = set() paths = get_addons_paths() + excluded_paths = get_excluded_paths() + paths = filter_valid_paths(paths, excluded_paths) print("Looking up for packages in {}".format(paths)) for addons_path in paths: for packages in addons_path.glob('**/apt-packages.txt'): print("Installing packages from %s" % packages) - with open(packages, 'r') as pack_file: - lines = [line.strip() for line in pack_file] + with packages.open('r') as pack_file: + lines = [ + six.ensure_text(line).strip() + for line in pack_file + if six.ensure_text(line).strip() + ] package_list.update(set(lines)) extras = os.environ.get('EXTRA_APT_PACKAGES', '') - print(f"Adding extra packages {extras}") + print("Adding extra packages {extras}".format(extras=extras)) if extras: for package in extras.split(','): - if not package: + if not package.strip(): continue - package_list.add(package) + package_list.add( + six.ensure_text(package).strip() + ) if len(package_list) > 0: - print(f"Installing {package_list}") + print("Installing {package_list}".format(package_list=package_list)) ret = pipe(['apt-get', 'update']) # Something went wrong, stop the service as it's failing diff --git a/12.0-nightly/Dockerfile b/12.0-nightly/Dockerfile index 23a24819e..2a5e7fbf3 100644 --- a/12.0-nightly/Dockerfile +++ b/12.0-nightly/Dockerfile @@ -5,89 +5,93 @@ LABEL maintainer="Archeti " ENV LANG C.UTF-8 ARG DEBIAN_FRONTEND=noninteractive +COPY ./platform.sh /platform.sh # Install some dependencies python3.7 RUN set -x; \ - apt-get update \ + export ARCH=`sh /platform.sh` \ + # Setup deb source for postgresql client + && apt-get update \ && apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg \ + python3.7 \ python3-wheel \ python3-setuptools \ python3-pip \ - # python3.7 \ - # libpython3.7 \ + python3-psycopg2 \ + python3-ldap \ + python3-libsass \ + python3-lxml \ + python3-pillow \ + python3-pypdf2 \ + python3-psutil \ + python3-asn1crypto \ curl \ - gnupg \ - libpq-dev \ - libsasl2-2 \ - libldap-2.4-2 \ - libxml2 \ - libxmlsec1 \ - libxslt1.1 \ + # libpq-dev \ + # libsasl2-2 \ + # libldap-2.4-2 \ + # libxml2 \ + # libxmlsec1 \ + # libxslt1.1 \ sudo \ node-less \ - # python3-yaml \ - && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 \ - # && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 2 \ - && update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1 \ + && echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ + && export GNUPGHOME="$(mktemp -d)" \ + && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ + && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ + && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ + && gpgconf --kill all \ + && rm -rf "$GNUPGHOME" \ + # Download deb for wkhtmltox + && echo "Downloading https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb" \ + && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + ./wkhtmltox.deb \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 \ && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 2 \ - && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb \ - && apt-get install -y --no-install-recommends ./wkhtmltox.deb \ && rm -rf /var/lib/apt/lists/* wkhtmltox.deb -# Install latest postgresql-client -RUN set -x; \ - echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ - && export GNUPGHOME="$(mktemp -d)" \ - && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ - && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ - && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ - && gpgconf --kill all \ - && rm -rf "$GNUPGHOME" \ - && apt-get update \ - && apt-get install -y postgresql-client \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH=/usr/local/bin:$PATH +ENV PATH=/var/lib/odoo/.local/bin:/usr/local/bin:$PATH # Install Odoo Including things from sources ENV ODOO_VERSION 12.0 -ENV ODOO_RELEASE=20210625 +ENV ODOO_RELEASE= ARG ODOO_ARCHIVE=odoo_${ODOO_VERSION}.${ODOO_RELEASE}.tar.gz RUN set -x; \ apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ + # build-essential \ + # python3.7-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ && pip install -U pip \ && /usr/bin/env pip install \ - psycogreen \ + odoo-tools \ \ - && curl -o requirements.txt https://raw.githubusercontent.com/odoo/odoo/${ODOO_VERSION}/requirements.txt \ - && pip install -r requirements.txt \ - && /usr/bin/env pip install https://nightly.odoo.com/${ODOO_VERSION}/nightly/src/${ODOO_ARCHIVE} \ + && odootools manage setup --release "" --repo "https://github.com/odoo/odoo.git" --ref "" "12.0" \ && cd / \ && apt-get --purge remove -y \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ + # build-essential \ + # python3.7-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* - -VOLUME /etc/odoo -VOLUME /var/lib/odoo + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /root/.cache -COPY ./odoo.conf /etc/odoo/ -COPY ./entrypoint.py / -COPY ./sudo-entrypoint.py / -COPY ./find_modules.py /scripts/find_modules.py -COPY ./prepare_project.py /scripts/prepare_project.py +# COPY ./odoo.conf /etc/odoo/ +# COPY ./entrypoint.py / +# COPY ./sudo-entrypoint.py / ARG UID=1000 ARG GID=1000 @@ -95,12 +99,15 @@ ARG GID=1000 RUN mkdir /addons \ && groupadd -r -g ${GID} odoo \ && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ - && chown odoo /etc/odoo/odoo.conf \ + # && chown odoo /etc/odoo/odoo.conf \ && chown -R odoo:odoo /addons \ - && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /sudo-entrypoint.py" >> /etc/sudoers + && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /usr/local/bin/odootools" >> /etc/sudoers + +# VOLUME /etc/odoo +VOLUME /var/lib/odoo -ENV ODOO_RC /etc/odoo/odoo.conf -ENV ODOO_BASE_PATH /usr/local/lib/python3.6/dist-packages/odoo/addons +ENV ODOO_RC /var/lib/odoo/odoo.conf +ENV ODOO_BASE_PATH /usr/local/lib/python3.7/dist-packages/odoo/addons # Env variable defined to monitor the kind of service running # it could be a staging/production/test or anything and undefined # is the default in case we need to know servers that aren't correctly @@ -113,19 +120,19 @@ expose 8071 USER odoo LABEL version="12.0" -LABEL release="20210625" +LABEL release="" -LABEL org.opencontainers.image.created="2021-06-25T17:55:41.200370" +LABEL org.opencontainers.image.created="2022-11-04T20:06:13.823004" LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" -LABEL org.opencontainers.image.authors="Archeti " +LABEL org.opencontainers.image.authors="OdooPlus " LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.version="12.0" -LABEL org.opencontainers.image.vendor="ArcheTI" +LABEL org.opencontainers.image.vendor="OdooPlus" LABEL org.opencontainers.image.ref.name="12.0-nightly" LABEL org.opencontainers.image.title="Odoo 12.0" LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." -ENTRYPOINT ["/entrypoint.py"] +ENTRYPOINT ["odootools", "--log-level", "INFO", "entrypoint", "user"] cmd ["odoo"] diff --git a/12.0-nightly/entrypoint.py b/12.0-nightly/entrypoint.py index f2ce4654b..bb0ffddc3 100755 --- a/12.0-nightly/entrypoint.py +++ b/12.0-nightly/entrypoint.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # import argparse +from __future__ import print_function import time import shlex import subprocess @@ -17,25 +18,9 @@ from passlib.context import CryptContext from contextlib import contextmanager -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - -try: - from pip._internal.network.session import PipSession - from pip._internal.req.req_file import parse_requirements - from pip._internal.req.constructors import ( - install_req_from_parsed_requirement - ) -except Exception: - from pip.download import PipSession - from pip.req.req_file import parse_requirements - - def install_req_from_parsed_requirement(req): - return req - -from collections import defaultdict +from odoo_tools.compat import Path +from odoo_tools.modules.search import find_addons_paths, find_modules_paths +from odoo_tools.configuration.pips import merge_requirements try: # python3 @@ -68,63 +53,6 @@ def quote(s): return "'" + s.replace("'", "'\"'\"'") + "'" -class Requirement(object): - def __init__(self): - self.extras = set() - self.specifiers = set() - self.links = set() - self.editable = False - - -def merge_requirements(files): - requirements = defaultdict(lambda: Requirement()) - links = set() - - for filename in files: - f_requirements = parse_requirements( - str(filename), session=PipSession() - ) - for parsed_requirement in f_requirements: - requirement = install_req_from_parsed_requirement( - parsed_requirement - ) - - if requirement.markers and not requirement.markers.evaluate(): - continue - - if not hasattr(requirement.req, 'name'): - links.add(requirement.link.url) - break - name = requirement.req.name - specifiers = requirement.req.specifier - extras = requirement.req.extras - requirements[name].extras |= set(extras) - requirements[name].specifiers |= set(specifiers) - if requirement.link: - requirements[name].links |= {requirement.link.url} - requirements[name].editable |= requirement.editable - - result = [] - for key, value in requirements.items(): - if value.links: - result.append("%s" % value.links.pop()) - elif not value.extras: - result.append( - "%s %s" % (key, ",".join(map(str, value.specifiers))) - ) - else: - result.append("%s [%s] %s" % ( - key, - ",".join(map(str, value.extras)), - ",".join(map(str, value.specifiers)) - )) - - for link in links: - result.append(link) - - return "\n".join(result) - - def pipe(args): """ Call the process with std(in,out,err) @@ -178,42 +106,12 @@ def call_sudo_entrypoint(): return ret -def get_all_manifests(): - odoo_manifests = glob.glob('/addons/**/__manifest__.py', recursive=True) - erp_manifest = glob.glob('/addons/**/__openerp__.py', recursive=True) - - return odoo_manifests + erp_manifest - - -def module_name(manifest_file): - return path.basename(path.dirname(manifest_file)) - - -def get_module(manifest_file): - with open(manifest_file) as manifest_in: - try: - code = compile(manifest_in.read(), '__manifest__', 'eval') - module = eval(code, {}, {}) - except Exception: - module = {} - - return module - - -def is_installable(manifest): - return manifest.get('installable', True) - - -def is_server_wide(manifest): - return manifest.get('server_wide', False) - - def get_server_wide_modules(manifests): base_server_wide_modules = ['base', 'web'] custom_server_wide_modules = [ - module_name(filename) - for filename, module in manifests.items() - if is_server_wide(module) + manifest.technical_name + for manifest in manifests + if manifest.server_wide ] return base_server_wide_modules + custom_server_wide_modules @@ -246,7 +144,12 @@ def install_python_dependencies(valid_paths): req_file = Path('/var/lib/odoo/requirements.txt') with req_file.open('w') as fout: data = merge_requirements(requirement_files) - fout.write(data) + data = "\n".join(data) + + try: + fout.write(data) + except Exception: + fout.write(data.decode('utf-8')) for requirements in requirement_files: print("Installing python packages from %s" % requirements) @@ -338,65 +241,44 @@ def get_extra_paths(): ] -def get_valid_paths(): - base_addons = os.environ.get('ODOO_BASE_PATH') +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] - # addons = os.listdir('/addons') - valid_paths = [base_addons] +def get_valid_paths(): + base_addons = os.environ['ODOO_BASE_PATH'] - addons_paths = get_dirs('/addons') - addons_paths += get_extra_paths() + base_addons_paths = [base_addons, '/addons'] + base_addons_paths += get_extra_paths() - for addons_path in addons_paths: - print("Lookup addons in %s" % addons_path) - flush_streams() - addons = get_dirs(addons_path) - for addon in addons: - files = os.listdir(addon) - if ( - '__init__.py' in files and - '__manifest__.py' in files or - '__openerp__.py' in files - ): - valid_paths.append(addons_path) - print("Addons found !") - flush_streams() - break - if addons_path in valid_paths: - continue - else: - print("No addons found in path. Skipping...") - flush_streams() + valid_paths = find_addons_paths(base_addons_paths) + + for valid_path in valid_paths: + print("Found installable modules in {}".format(valid_path)) return valid_paths def setup_addons_paths(config, valid_paths): - config.set('options', 'addons_path', ",".join(valid_paths)) + config.set('options', 'addons_path', ",".join([ + str(path) for path in valid_paths + ])) flush_streams() -def setup_server_wide_modules(config): +def setup_server_wide_modules(config, valid_paths): print("Searching for server wide modules") - all_manifests = get_all_manifests() + filters = set(['installable']) + all_manifests = find_modules_paths(valid_paths, filters) - print("Found %s modules" % (len(all_manifests))) + print("Found %s installable modules " % (len(all_manifests),)) - manifests = { - filename: get_module(filename) - for filename in all_manifests - } - - installable_manifests = { - filename: module - for filename, module in manifests.items() - if is_installable(module) - } - - print("Found %s installable modules " % (len(installable_manifests),)) - - server_wide_modules = get_server_wide_modules(installable_manifests) + server_wide_modules = get_server_wide_modules(all_manifests) if len(server_wide_modules) > 2: modules = ",".join(server_wide_modules) @@ -408,6 +290,36 @@ def setup_server_wide_modules(config): flush_streams() +def convert_value(name, value): + if value in ['True', 'False']: + return value == 'True' + + return value + + +def setup_env_config(config): + try: + from odoo.tools import config as odoo_config + except ImportError: + from openerp.tools import config as odoo_config + + params_by_name = {} + for key, opt in odoo_config.casts.items(): + value_opt = opt.get_opt_string() + value_opt = value_opt.upper().replace('--', 'ODOO_') + value_opt = value_opt.replace('-', '_') + params_by_name[value_opt] = key + + for key, value in os.environ.items(): + if not key.startswith('ODOO_'): + continue + + if key in params_by_name: + config_name = params_by_name[key] + converted_value = convert_value(key, value) + config.set('options', config_name, converted_value) + + def setup_environ(config): print("Configuring environment variables for postgresql") @@ -499,12 +411,39 @@ def wait_postgresql(): @contextmanager def get_config(config_path): config = ConfigParser() - config.read(config_path) + config.read(str(config_path)) yield config - with config_path.open('w') as out: - config.write(out) + try: + with config_path.open('w') as out: + config.write(out) + except Exception: + with config_path.open('wb') as out: + config.write(out) + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for vpath in paths: + cur_path = Path(vpath) + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(vpath) + + return res_paths def main(): @@ -518,7 +457,9 @@ def main(): sys.exit(ret) # Install python packages with pip in user home - valid_paths = get_valid_paths() + orig_valid_paths = get_valid_paths() + excluded_paths = get_excluded_paths() + valid_paths = filter_valid_paths(orig_valid_paths, excluded_paths) config_path = Path(os.environ.get('ODOO_RC', '/etc/odoo/odoo.cfg')) @@ -528,7 +469,8 @@ def main(): install_master_password(config) setup_environ(config) setup_addons_paths(config, valid_paths) - setup_server_wide_modules(config) + setup_server_wide_modules(config, valid_paths) + setup_env_config(config) if not os.environ.get('ODOO_SKIP_POSTGRES_WAIT'): wait_postgresql() diff --git a/12.0-nightly/find_modules.py b/12.0-nightly/find_modules.py deleted file mode 100644 index 9cb943748..000000000 --- a/12.0-nightly/find_modules.py +++ /dev/null @@ -1,76 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -from itertools import chain - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - "-p", - dest='paths', - action="append", - help="Location in which to search", - default=[] - ) - - parser.add_option( - "--only-name", - dest="only_name", - action="store_true", - help="Only display module name instead of path", - default=False, - ) - - parser.add_option( - '--csv', - dest="is_csv", - action="store_true", - help="Output as a comma separated list", - default=False - ) - - return parser - - -def find_modules(options, path): - modules = set() - - path = Path.cwd() / path - - erp_manifest = '__openerp__.py' - odoo_manifest = '__manifest__.py' - - manifest_globs = chain( - path.glob('**/{}'.format(erp_manifest)), - path.glob('**/{}'.format(odoo_manifest)), - ) - - for path in manifest_globs: - rel_path = path.parent.relative_to(Path.cwd()) - if options.only_name: - modules.add(rel_path.name) - else: - modules.add(str(rel_path)) - - return modules - - -def main(options, args): - modules = set() - for path in options.paths: - modules = modules.union(find_modules(options, path)) - - return modules - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - modules = main(options, args) - - if not options.is_csv: - for module in modules: - print(module) - else: - print(",".join(modules), end="") diff --git a/12.0-nightly/platform.sh b/12.0-nightly/platform.sh new file mode 100644 index 000000000..fdf5e42eb --- /dev/null +++ b/12.0-nightly/platform.sh @@ -0,0 +1,13 @@ +PLATFORM=`uname -m` + +if [ $PLATFORM = "aarch64" ] +then + echo -n "arm64" +else +if [ $PLATFORM = "x86_64" ] +then + echo -n "amd64" +else + echo -n $PLATFORM +fi +fi diff --git a/12.0-nightly/prepare_project.py b/12.0-nightly/prepare_project.py deleted file mode 100644 index 74a53f2e0..000000000 --- a/12.0-nightly/prepare_project.py +++ /dev/null @@ -1,198 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -import toml -from giturlparse import parse -from contextlib import contextmanager -import os -from subprocess import run -from urllib.parse import urlparse - - -def get_services(services): - return { - service.get('name'): service - for service in services.get('services') - } - - -def addons_by_project(options, addons): - res = {} - - for addon in addons: - if addon['url'] == 'self': - if options.ignore_self: - continue - parsed = parse(options.url) - else: - parsed = parse(addon['url']) - - auth = parsed.protocol in ['git', 'ssh'] - - res[parsed.repo] = dict( - addon, - url=parsed.url2https, - auth=auth - ) - - return res - - -def merge_addons(options, base, other): - base_addons = addons_by_project(options, base) - other_addons = addons_by_project(options, other) - - for name, addon in other_addons.items(): - if name not in base_addons: - base_addons[name] = addon - else: - base_addons[name] = dict(base_addons[name], **addon) - - return [ - addon - for addon in base_addons.values() - ] - - -def merge_services(options, base, other): - basic_inherit = dict(base, **other) - - if base.get('addons') or other.get('addons'): - basic_inherit['addons'] = merge_addons( - options, - base.get('addons', []), - other.get('addons', []) - ) - - return basic_inherit - - -def compile_service(options, services, name): - service = services.get(name, {}) - if 'inherit' in service: - merge_service = compile_service(options, services, service['inherit']) - service = merge_services(options, merge_service, service) - - return service - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - '-f', - '--file', - dest="file", - help="Input File" - ) - - parser.add_option( - '--url', - dest='url', - help="Url of self project" - ) - - parser.add_option( - '-o', - '--output', - dest="output_directory", - help="Output Directory" - ) - - parser.add_option( - '-e', - dest="env", - help="Environment to prepare" - ) - - parser.add_option( - '--username', - dest="username", - help="Username to replace with", - ) - - parser.add_option( - '--password', - dest="password", - help="password to set on https urls" - ) - - parser.add_option( - '-b', - '--branch', - dest="branch", - help="Default branch if no ref is defined" - ) - - parser.add_option( - '--ignore-self', - dest="ignore_self", - action="store_true", - help="Ignore self url as it's already fetched", - default=False - ) - - return parser - - -@contextmanager -def cd(directory): - cwd = Path.cwd() - try: - os.chdir(directory) - yield - finally: - os.chdir(cwd) - - -def fetch_addons(options, addon): - parsed = parse(addon['url']) - - url = urlparse(parsed.url2https) - - if addon['auth'] and options.username and options.password: - url = url._replace( - netloc="{}:{}@{}".format( - options.username, - options.password, - url.netloc - ) - ) - - repo_path = Path.cwd() / options.output_directory / parsed.repo - - repo_path.mkdir(exist_ok=True) - - with cd(repo_path): - run(['git', 'init']) - run(['git', 'remote', 'add', 'origin', url.geturl()]) - - ref = addon.get('commit') or addon.get('branch') or options.branch - - if ref: - run(['git', 'fetch', 'origin', ref]) - else: - run(['git', 'fetch', 'origin']) - - run(['git', 'checkout', 'FETCH_HEAD']) - run(['git', 'remote', 'remove', 'origin']) - - -def main(options, args): - with Path(options.file).open('r') as fin: - services = toml.loads(fin.read()) - - by_name = get_services(services) - - outputdir = Path(options.output_directory) - outputdir.mkdir(exist_ok=True) - - service = compile_service(options, by_name, options.env) - - for addons in service.get('addons', []): - fetch_addons(options, addons) - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - main(options, args) diff --git a/12.0-nightly/sudo-entrypoint.py b/12.0-nightly/sudo-entrypoint.py index 98779baae..916239170 100755 --- a/12.0-nightly/sudo-entrypoint.py +++ b/12.0-nightly/sudo-entrypoint.py @@ -12,6 +12,7 @@ from os import path from os.path import expanduser import shutil +import six try: from pathlib import Path @@ -69,6 +70,37 @@ def get_addons_paths(): ] +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for cur_path in paths: + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(cur_path) + + return res_paths + + def install_apt_packages(): """ Install debian dependencies. @@ -76,26 +108,34 @@ def install_apt_packages(): package_list = set() paths = get_addons_paths() + excluded_paths = get_excluded_paths() + paths = filter_valid_paths(paths, excluded_paths) print("Looking up for packages in {}".format(paths)) for addons_path in paths: for packages in addons_path.glob('**/apt-packages.txt'): print("Installing packages from %s" % packages) - with open(packages, 'r') as pack_file: - lines = [line.strip() for line in pack_file] + with packages.open('r') as pack_file: + lines = [ + six.ensure_text(line).strip() + for line in pack_file + if six.ensure_text(line).strip() + ] package_list.update(set(lines)) extras = os.environ.get('EXTRA_APT_PACKAGES', '') - print(f"Adding extra packages {extras}") + print("Adding extra packages {extras}".format(extras=extras)) if extras: for package in extras.split(','): - if not package: + if not package.strip(): continue - package_list.add(package) + package_list.add( + six.ensure_text(package).strip() + ) if len(package_list) > 0: - print(f"Installing {package_list}") + print("Installing {package_list}".format(package_list=package_list)) ret = pipe(['apt-get', 'update']) # Something went wrong, stop the service as it's failing diff --git a/12.0/Dockerfile b/12.0/Dockerfile index 6438f2076..3687b9c8f 100644 --- a/12.0/Dockerfile +++ b/12.0/Dockerfile @@ -5,89 +5,93 @@ LABEL maintainer="Archeti " ENV LANG C.UTF-8 ARG DEBIAN_FRONTEND=noninteractive +COPY ./platform.sh /platform.sh # Install some dependencies python3.7 RUN set -x; \ - apt-get update \ + export ARCH=`sh /platform.sh` \ + # Setup deb source for postgresql client + && apt-get update \ && apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg \ + python3.7 \ python3-wheel \ python3-setuptools \ python3-pip \ - # python3.7 \ - # libpython3.7 \ + python3-psycopg2 \ + python3-ldap \ + python3-libsass \ + python3-lxml \ + python3-pillow \ + python3-pypdf2 \ + python3-psutil \ + python3-asn1crypto \ curl \ - gnupg \ - libpq-dev \ - libsasl2-2 \ - libldap-2.4-2 \ - libxml2 \ - libxmlsec1 \ - libxslt1.1 \ + # libpq-dev \ + # libsasl2-2 \ + # libldap-2.4-2 \ + # libxml2 \ + # libxmlsec1 \ + # libxslt1.1 \ sudo \ node-less \ - # python3-yaml \ - && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 \ - # && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 2 \ - && update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1 \ + && echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ + && export GNUPGHOME="$(mktemp -d)" \ + && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ + && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ + && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ + && gpgconf --kill all \ + && rm -rf "$GNUPGHOME" \ + # Download deb for wkhtmltox + && echo "Downloading https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb" \ + && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + ./wkhtmltox.deb \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 \ && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 2 \ - && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb \ - && apt-get install -y --no-install-recommends ./wkhtmltox.deb \ && rm -rf /var/lib/apt/lists/* wkhtmltox.deb -# Install latest postgresql-client -RUN set -x; \ - echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ - && export GNUPGHOME="$(mktemp -d)" \ - && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ - && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ - && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ - && gpgconf --kill all \ - && rm -rf "$GNUPGHOME" \ - && apt-get update \ - && apt-get install -y postgresql-client \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH=/usr/local/bin:$PATH +ENV PATH=/var/lib/odoo/.local/bin:/usr/local/bin:$PATH # Install Odoo Including things from sources ENV ODOO_VERSION 12.0 -ENV ODOO_RELEASE=20200623 +ENV ODOO_RELEASE=20211011 ARG ODOO_ARCHIVE=odoo_${ODOO_VERSION}.${ODOO_RELEASE}.tar.gz RUN set -x; \ apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ + # build-essential \ + # python3.7-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ && pip install -U pip \ && /usr/bin/env pip install \ - psycogreen \ + odoo-tools \ \ - && curl -o requirements.txt https://raw.githubusercontent.com/odoo/odoo/${ODOO_VERSION}/requirements.txt \ - && pip install -r requirements.txt \ - && /usr/bin/env pip install https://nightly.odoo.com/${ODOO_VERSION}/nightly/src/${ODOO_ARCHIVE} \ + && odootools manage setup --release "20211011" --repo "https://github.com/odoo/odoo.git" --ref "" "12.0" \ && cd / \ && apt-get --purge remove -y \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ + # build-essential \ + # python3.7-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* - -VOLUME /etc/odoo -VOLUME /var/lib/odoo + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /root/.cache -COPY ./odoo.conf /etc/odoo/ -COPY ./entrypoint.py / -COPY ./sudo-entrypoint.py / -COPY ./find_modules.py /scripts/find_modules.py -COPY ./prepare_project.py /scripts/prepare_project.py +# COPY ./odoo.conf /etc/odoo/ +# COPY ./entrypoint.py / +# COPY ./sudo-entrypoint.py / ARG UID=1000 ARG GID=1000 @@ -95,12 +99,15 @@ ARG GID=1000 RUN mkdir /addons \ && groupadd -r -g ${GID} odoo \ && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ - && chown odoo /etc/odoo/odoo.conf \ + # && chown odoo /etc/odoo/odoo.conf \ && chown -R odoo:odoo /addons \ - && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /sudo-entrypoint.py" >> /etc/sudoers + && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /usr/local/bin/odootools" >> /etc/sudoers + +# VOLUME /etc/odoo +VOLUME /var/lib/odoo -ENV ODOO_RC /etc/odoo/odoo.conf -ENV ODOO_BASE_PATH /usr/local/lib/python3.6/dist-packages/odoo/addons +ENV ODOO_RC /var/lib/odoo/odoo.conf +ENV ODOO_BASE_PATH /usr/local/lib/python3.7/dist-packages/odoo/addons # Env variable defined to monitor the kind of service running # it could be a staging/production/test or anything and undefined # is the default in case we need to know servers that aren't correctly @@ -113,19 +120,19 @@ expose 8071 USER odoo LABEL version="12.0" -LABEL release="20200623" +LABEL release="20211011" -LABEL org.opencontainers.image.created="2021-06-25T17:55:41.195800" +LABEL org.opencontainers.image.created="2022-11-04T20:06:13.802326" LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" -LABEL org.opencontainers.image.authors="Archeti " +LABEL org.opencontainers.image.authors="OdooPlus " LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.version="12.0" -LABEL org.opencontainers.image.vendor="ArcheTI" +LABEL org.opencontainers.image.vendor="OdooPlus" LABEL org.opencontainers.image.ref.name="12.0" LABEL org.opencontainers.image.title="Odoo 12.0" LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." -ENTRYPOINT ["/entrypoint.py"] +ENTRYPOINT ["odootools", "--log-level", "INFO", "entrypoint", "user"] cmd ["odoo"] diff --git a/12.0/entrypoint.py b/12.0/entrypoint.py index f2ce4654b..bb0ffddc3 100755 --- a/12.0/entrypoint.py +++ b/12.0/entrypoint.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # import argparse +from __future__ import print_function import time import shlex import subprocess @@ -17,25 +18,9 @@ from passlib.context import CryptContext from contextlib import contextmanager -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - -try: - from pip._internal.network.session import PipSession - from pip._internal.req.req_file import parse_requirements - from pip._internal.req.constructors import ( - install_req_from_parsed_requirement - ) -except Exception: - from pip.download import PipSession - from pip.req.req_file import parse_requirements - - def install_req_from_parsed_requirement(req): - return req - -from collections import defaultdict +from odoo_tools.compat import Path +from odoo_tools.modules.search import find_addons_paths, find_modules_paths +from odoo_tools.configuration.pips import merge_requirements try: # python3 @@ -68,63 +53,6 @@ def quote(s): return "'" + s.replace("'", "'\"'\"'") + "'" -class Requirement(object): - def __init__(self): - self.extras = set() - self.specifiers = set() - self.links = set() - self.editable = False - - -def merge_requirements(files): - requirements = defaultdict(lambda: Requirement()) - links = set() - - for filename in files: - f_requirements = parse_requirements( - str(filename), session=PipSession() - ) - for parsed_requirement in f_requirements: - requirement = install_req_from_parsed_requirement( - parsed_requirement - ) - - if requirement.markers and not requirement.markers.evaluate(): - continue - - if not hasattr(requirement.req, 'name'): - links.add(requirement.link.url) - break - name = requirement.req.name - specifiers = requirement.req.specifier - extras = requirement.req.extras - requirements[name].extras |= set(extras) - requirements[name].specifiers |= set(specifiers) - if requirement.link: - requirements[name].links |= {requirement.link.url} - requirements[name].editable |= requirement.editable - - result = [] - for key, value in requirements.items(): - if value.links: - result.append("%s" % value.links.pop()) - elif not value.extras: - result.append( - "%s %s" % (key, ",".join(map(str, value.specifiers))) - ) - else: - result.append("%s [%s] %s" % ( - key, - ",".join(map(str, value.extras)), - ",".join(map(str, value.specifiers)) - )) - - for link in links: - result.append(link) - - return "\n".join(result) - - def pipe(args): """ Call the process with std(in,out,err) @@ -178,42 +106,12 @@ def call_sudo_entrypoint(): return ret -def get_all_manifests(): - odoo_manifests = glob.glob('/addons/**/__manifest__.py', recursive=True) - erp_manifest = glob.glob('/addons/**/__openerp__.py', recursive=True) - - return odoo_manifests + erp_manifest - - -def module_name(manifest_file): - return path.basename(path.dirname(manifest_file)) - - -def get_module(manifest_file): - with open(manifest_file) as manifest_in: - try: - code = compile(manifest_in.read(), '__manifest__', 'eval') - module = eval(code, {}, {}) - except Exception: - module = {} - - return module - - -def is_installable(manifest): - return manifest.get('installable', True) - - -def is_server_wide(manifest): - return manifest.get('server_wide', False) - - def get_server_wide_modules(manifests): base_server_wide_modules = ['base', 'web'] custom_server_wide_modules = [ - module_name(filename) - for filename, module in manifests.items() - if is_server_wide(module) + manifest.technical_name + for manifest in manifests + if manifest.server_wide ] return base_server_wide_modules + custom_server_wide_modules @@ -246,7 +144,12 @@ def install_python_dependencies(valid_paths): req_file = Path('/var/lib/odoo/requirements.txt') with req_file.open('w') as fout: data = merge_requirements(requirement_files) - fout.write(data) + data = "\n".join(data) + + try: + fout.write(data) + except Exception: + fout.write(data.decode('utf-8')) for requirements in requirement_files: print("Installing python packages from %s" % requirements) @@ -338,65 +241,44 @@ def get_extra_paths(): ] -def get_valid_paths(): - base_addons = os.environ.get('ODOO_BASE_PATH') +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] - # addons = os.listdir('/addons') - valid_paths = [base_addons] +def get_valid_paths(): + base_addons = os.environ['ODOO_BASE_PATH'] - addons_paths = get_dirs('/addons') - addons_paths += get_extra_paths() + base_addons_paths = [base_addons, '/addons'] + base_addons_paths += get_extra_paths() - for addons_path in addons_paths: - print("Lookup addons in %s" % addons_path) - flush_streams() - addons = get_dirs(addons_path) - for addon in addons: - files = os.listdir(addon) - if ( - '__init__.py' in files and - '__manifest__.py' in files or - '__openerp__.py' in files - ): - valid_paths.append(addons_path) - print("Addons found !") - flush_streams() - break - if addons_path in valid_paths: - continue - else: - print("No addons found in path. Skipping...") - flush_streams() + valid_paths = find_addons_paths(base_addons_paths) + + for valid_path in valid_paths: + print("Found installable modules in {}".format(valid_path)) return valid_paths def setup_addons_paths(config, valid_paths): - config.set('options', 'addons_path', ",".join(valid_paths)) + config.set('options', 'addons_path', ",".join([ + str(path) for path in valid_paths + ])) flush_streams() -def setup_server_wide_modules(config): +def setup_server_wide_modules(config, valid_paths): print("Searching for server wide modules") - all_manifests = get_all_manifests() + filters = set(['installable']) + all_manifests = find_modules_paths(valid_paths, filters) - print("Found %s modules" % (len(all_manifests))) + print("Found %s installable modules " % (len(all_manifests),)) - manifests = { - filename: get_module(filename) - for filename in all_manifests - } - - installable_manifests = { - filename: module - for filename, module in manifests.items() - if is_installable(module) - } - - print("Found %s installable modules " % (len(installable_manifests),)) - - server_wide_modules = get_server_wide_modules(installable_manifests) + server_wide_modules = get_server_wide_modules(all_manifests) if len(server_wide_modules) > 2: modules = ",".join(server_wide_modules) @@ -408,6 +290,36 @@ def setup_server_wide_modules(config): flush_streams() +def convert_value(name, value): + if value in ['True', 'False']: + return value == 'True' + + return value + + +def setup_env_config(config): + try: + from odoo.tools import config as odoo_config + except ImportError: + from openerp.tools import config as odoo_config + + params_by_name = {} + for key, opt in odoo_config.casts.items(): + value_opt = opt.get_opt_string() + value_opt = value_opt.upper().replace('--', 'ODOO_') + value_opt = value_opt.replace('-', '_') + params_by_name[value_opt] = key + + for key, value in os.environ.items(): + if not key.startswith('ODOO_'): + continue + + if key in params_by_name: + config_name = params_by_name[key] + converted_value = convert_value(key, value) + config.set('options', config_name, converted_value) + + def setup_environ(config): print("Configuring environment variables for postgresql") @@ -499,12 +411,39 @@ def wait_postgresql(): @contextmanager def get_config(config_path): config = ConfigParser() - config.read(config_path) + config.read(str(config_path)) yield config - with config_path.open('w') as out: - config.write(out) + try: + with config_path.open('w') as out: + config.write(out) + except Exception: + with config_path.open('wb') as out: + config.write(out) + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for vpath in paths: + cur_path = Path(vpath) + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(vpath) + + return res_paths def main(): @@ -518,7 +457,9 @@ def main(): sys.exit(ret) # Install python packages with pip in user home - valid_paths = get_valid_paths() + orig_valid_paths = get_valid_paths() + excluded_paths = get_excluded_paths() + valid_paths = filter_valid_paths(orig_valid_paths, excluded_paths) config_path = Path(os.environ.get('ODOO_RC', '/etc/odoo/odoo.cfg')) @@ -528,7 +469,8 @@ def main(): install_master_password(config) setup_environ(config) setup_addons_paths(config, valid_paths) - setup_server_wide_modules(config) + setup_server_wide_modules(config, valid_paths) + setup_env_config(config) if not os.environ.get('ODOO_SKIP_POSTGRES_WAIT'): wait_postgresql() diff --git a/12.0/find_modules.py b/12.0/find_modules.py deleted file mode 100644 index 9cb943748..000000000 --- a/12.0/find_modules.py +++ /dev/null @@ -1,76 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -from itertools import chain - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - "-p", - dest='paths', - action="append", - help="Location in which to search", - default=[] - ) - - parser.add_option( - "--only-name", - dest="only_name", - action="store_true", - help="Only display module name instead of path", - default=False, - ) - - parser.add_option( - '--csv', - dest="is_csv", - action="store_true", - help="Output as a comma separated list", - default=False - ) - - return parser - - -def find_modules(options, path): - modules = set() - - path = Path.cwd() / path - - erp_manifest = '__openerp__.py' - odoo_manifest = '__manifest__.py' - - manifest_globs = chain( - path.glob('**/{}'.format(erp_manifest)), - path.glob('**/{}'.format(odoo_manifest)), - ) - - for path in manifest_globs: - rel_path = path.parent.relative_to(Path.cwd()) - if options.only_name: - modules.add(rel_path.name) - else: - modules.add(str(rel_path)) - - return modules - - -def main(options, args): - modules = set() - for path in options.paths: - modules = modules.union(find_modules(options, path)) - - return modules - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - modules = main(options, args) - - if not options.is_csv: - for module in modules: - print(module) - else: - print(",".join(modules), end="") diff --git a/12.0/platform.sh b/12.0/platform.sh new file mode 100644 index 000000000..fdf5e42eb --- /dev/null +++ b/12.0/platform.sh @@ -0,0 +1,13 @@ +PLATFORM=`uname -m` + +if [ $PLATFORM = "aarch64" ] +then + echo -n "arm64" +else +if [ $PLATFORM = "x86_64" ] +then + echo -n "amd64" +else + echo -n $PLATFORM +fi +fi diff --git a/12.0/prepare_project.py b/12.0/prepare_project.py deleted file mode 100644 index 74a53f2e0..000000000 --- a/12.0/prepare_project.py +++ /dev/null @@ -1,198 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -import toml -from giturlparse import parse -from contextlib import contextmanager -import os -from subprocess import run -from urllib.parse import urlparse - - -def get_services(services): - return { - service.get('name'): service - for service in services.get('services') - } - - -def addons_by_project(options, addons): - res = {} - - for addon in addons: - if addon['url'] == 'self': - if options.ignore_self: - continue - parsed = parse(options.url) - else: - parsed = parse(addon['url']) - - auth = parsed.protocol in ['git', 'ssh'] - - res[parsed.repo] = dict( - addon, - url=parsed.url2https, - auth=auth - ) - - return res - - -def merge_addons(options, base, other): - base_addons = addons_by_project(options, base) - other_addons = addons_by_project(options, other) - - for name, addon in other_addons.items(): - if name not in base_addons: - base_addons[name] = addon - else: - base_addons[name] = dict(base_addons[name], **addon) - - return [ - addon - for addon in base_addons.values() - ] - - -def merge_services(options, base, other): - basic_inherit = dict(base, **other) - - if base.get('addons') or other.get('addons'): - basic_inherit['addons'] = merge_addons( - options, - base.get('addons', []), - other.get('addons', []) - ) - - return basic_inherit - - -def compile_service(options, services, name): - service = services.get(name, {}) - if 'inherit' in service: - merge_service = compile_service(options, services, service['inherit']) - service = merge_services(options, merge_service, service) - - return service - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - '-f', - '--file', - dest="file", - help="Input File" - ) - - parser.add_option( - '--url', - dest='url', - help="Url of self project" - ) - - parser.add_option( - '-o', - '--output', - dest="output_directory", - help="Output Directory" - ) - - parser.add_option( - '-e', - dest="env", - help="Environment to prepare" - ) - - parser.add_option( - '--username', - dest="username", - help="Username to replace with", - ) - - parser.add_option( - '--password', - dest="password", - help="password to set on https urls" - ) - - parser.add_option( - '-b', - '--branch', - dest="branch", - help="Default branch if no ref is defined" - ) - - parser.add_option( - '--ignore-self', - dest="ignore_self", - action="store_true", - help="Ignore self url as it's already fetched", - default=False - ) - - return parser - - -@contextmanager -def cd(directory): - cwd = Path.cwd() - try: - os.chdir(directory) - yield - finally: - os.chdir(cwd) - - -def fetch_addons(options, addon): - parsed = parse(addon['url']) - - url = urlparse(parsed.url2https) - - if addon['auth'] and options.username and options.password: - url = url._replace( - netloc="{}:{}@{}".format( - options.username, - options.password, - url.netloc - ) - ) - - repo_path = Path.cwd() / options.output_directory / parsed.repo - - repo_path.mkdir(exist_ok=True) - - with cd(repo_path): - run(['git', 'init']) - run(['git', 'remote', 'add', 'origin', url.geturl()]) - - ref = addon.get('commit') or addon.get('branch') or options.branch - - if ref: - run(['git', 'fetch', 'origin', ref]) - else: - run(['git', 'fetch', 'origin']) - - run(['git', 'checkout', 'FETCH_HEAD']) - run(['git', 'remote', 'remove', 'origin']) - - -def main(options, args): - with Path(options.file).open('r') as fin: - services = toml.loads(fin.read()) - - by_name = get_services(services) - - outputdir = Path(options.output_directory) - outputdir.mkdir(exist_ok=True) - - service = compile_service(options, by_name, options.env) - - for addons in service.get('addons', []): - fetch_addons(options, addons) - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - main(options, args) diff --git a/12.0/sudo-entrypoint.py b/12.0/sudo-entrypoint.py index 98779baae..916239170 100755 --- a/12.0/sudo-entrypoint.py +++ b/12.0/sudo-entrypoint.py @@ -12,6 +12,7 @@ from os import path from os.path import expanduser import shutil +import six try: from pathlib import Path @@ -69,6 +70,37 @@ def get_addons_paths(): ] +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for cur_path in paths: + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(cur_path) + + return res_paths + + def install_apt_packages(): """ Install debian dependencies. @@ -76,26 +108,34 @@ def install_apt_packages(): package_list = set() paths = get_addons_paths() + excluded_paths = get_excluded_paths() + paths = filter_valid_paths(paths, excluded_paths) print("Looking up for packages in {}".format(paths)) for addons_path in paths: for packages in addons_path.glob('**/apt-packages.txt'): print("Installing packages from %s" % packages) - with open(packages, 'r') as pack_file: - lines = [line.strip() for line in pack_file] + with packages.open('r') as pack_file: + lines = [ + six.ensure_text(line).strip() + for line in pack_file + if six.ensure_text(line).strip() + ] package_list.update(set(lines)) extras = os.environ.get('EXTRA_APT_PACKAGES', '') - print(f"Adding extra packages {extras}") + print("Adding extra packages {extras}".format(extras=extras)) if extras: for package in extras.split(','): - if not package: + if not package.strip(): continue - package_list.add(package) + package_list.add( + six.ensure_text(package).strip() + ) if len(package_list) > 0: - print(f"Installing {package_list}") + print("Installing {package_list}".format(package_list=package_list)) ret = pipe(['apt-get', 'update']) # Something went wrong, stop the service as it's failing diff --git a/13.0-nightly/Dockerfile b/13.0-nightly/Dockerfile index 9f380d82f..78bc9f1d4 100644 --- a/13.0-nightly/Dockerfile +++ b/13.0-nightly/Dockerfile @@ -5,89 +5,93 @@ LABEL maintainer="Archeti " ENV LANG C.UTF-8 ARG DEBIAN_FRONTEND=noninteractive +COPY ./platform.sh /platform.sh # Install some dependencies python3.7 RUN set -x; \ - apt-get update \ + export ARCH=`sh /platform.sh` \ + # Setup deb source for postgresql client + && apt-get update \ && apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg \ + python3.7 \ python3-wheel \ python3-setuptools \ python3-pip \ - # python3.7 \ - # libpython3.7 \ + python3-psycopg2 \ + python3-ldap \ + python3-libsass \ + python3-lxml \ + python3-pillow \ + python3-pypdf2 \ + python3-psutil \ + python3-asn1crypto \ curl \ - gnupg \ - libpq-dev \ - libsasl2-2 \ - libldap-2.4-2 \ - libxml2 \ - libxmlsec1 \ - libxslt1.1 \ + # libpq-dev \ + # libsasl2-2 \ + # libldap-2.4-2 \ + # libxml2 \ + # libxmlsec1 \ + # libxslt1.1 \ sudo \ node-less \ - # python3-yaml \ - && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 \ - # && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 2 \ - && update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1 \ + && echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ + && export GNUPGHOME="$(mktemp -d)" \ + && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ + && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ + && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ + && gpgconf --kill all \ + && rm -rf "$GNUPGHOME" \ + # Download deb for wkhtmltox + && echo "Downloading https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb" \ + && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + ./wkhtmltox.deb \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 \ && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 2 \ - && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb \ - && apt-get install -y --no-install-recommends ./wkhtmltox.deb \ && rm -rf /var/lib/apt/lists/* wkhtmltox.deb -# Install latest postgresql-client -RUN set -x; \ - echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ - && export GNUPGHOME="$(mktemp -d)" \ - && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ - && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ - && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ - && gpgconf --kill all \ - && rm -rf "$GNUPGHOME" \ - && apt-get update \ - && apt-get install -y postgresql-client \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH=/usr/local/bin:$PATH +ENV PATH=/var/lib/odoo/.local/bin:/usr/local/bin:$PATH # Install Odoo Including things from sources ENV ODOO_VERSION 13.0 -ENV ODOO_RELEASE=20210625 +ENV ODOO_RELEASE= ARG ODOO_ARCHIVE=odoo_${ODOO_VERSION}.${ODOO_RELEASE}.tar.gz RUN set -x; \ apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ + # build-essential \ + # python3.7-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ && pip install -U pip \ && /usr/bin/env pip install \ - psycogreen \ + odoo-tools \ \ - && curl -o requirements.txt https://raw.githubusercontent.com/odoo/odoo/${ODOO_VERSION}/requirements.txt \ - && pip install -r requirements.txt \ - && /usr/bin/env pip install https://nightly.odoo.com/${ODOO_VERSION}/nightly/src/${ODOO_ARCHIVE} \ + && odootools manage setup --release "" --repo "https://github.com/odoo/odoo.git" --ref "" "13.0" \ && cd / \ && apt-get --purge remove -y \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ + # build-essential \ + # python3.7-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* - -VOLUME /etc/odoo -VOLUME /var/lib/odoo + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /root/.cache -COPY ./odoo.conf /etc/odoo/ -COPY ./entrypoint.py / -COPY ./sudo-entrypoint.py / -COPY ./find_modules.py /scripts/find_modules.py -COPY ./prepare_project.py /scripts/prepare_project.py +# COPY ./odoo.conf /etc/odoo/ +# COPY ./entrypoint.py / +# COPY ./sudo-entrypoint.py / ARG UID=1000 ARG GID=1000 @@ -95,12 +99,15 @@ ARG GID=1000 RUN mkdir /addons \ && groupadd -r -g ${GID} odoo \ && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ - && chown odoo /etc/odoo/odoo.conf \ + # && chown odoo /etc/odoo/odoo.conf \ && chown -R odoo:odoo /addons \ - && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /sudo-entrypoint.py" >> /etc/sudoers + && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /usr/local/bin/odootools" >> /etc/sudoers + +# VOLUME /etc/odoo +VOLUME /var/lib/odoo -ENV ODOO_RC /etc/odoo/odoo.conf -ENV ODOO_BASE_PATH /usr/local/lib/python3.6/dist-packages/odoo/addons +ENV ODOO_RC /var/lib/odoo/odoo.conf +ENV ODOO_BASE_PATH /usr/local/lib/python3.7/dist-packages/odoo/addons # Env variable defined to monitor the kind of service running # it could be a staging/production/test or anything and undefined # is the default in case we need to know servers that aren't correctly @@ -113,19 +120,19 @@ expose 8071 USER odoo LABEL version="13.0" -LABEL release="20210625" +LABEL release="" -LABEL org.opencontainers.image.created="2021-06-25T17:55:41.201333" +LABEL org.opencontainers.image.created="2022-11-04T20:06:13.823936" LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" -LABEL org.opencontainers.image.authors="Archeti " +LABEL org.opencontainers.image.authors="OdooPlus " LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.version="13.0" -LABEL org.opencontainers.image.vendor="ArcheTI" +LABEL org.opencontainers.image.vendor="OdooPlus" LABEL org.opencontainers.image.ref.name="13.0-nightly" LABEL org.opencontainers.image.title="Odoo 13.0" LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." -ENTRYPOINT ["/entrypoint.py"] +ENTRYPOINT ["odootools", "--log-level", "INFO", "entrypoint", "user"] cmd ["odoo"] diff --git a/13.0-nightly/entrypoint.py b/13.0-nightly/entrypoint.py index f2ce4654b..bb0ffddc3 100755 --- a/13.0-nightly/entrypoint.py +++ b/13.0-nightly/entrypoint.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # import argparse +from __future__ import print_function import time import shlex import subprocess @@ -17,25 +18,9 @@ from passlib.context import CryptContext from contextlib import contextmanager -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - -try: - from pip._internal.network.session import PipSession - from pip._internal.req.req_file import parse_requirements - from pip._internal.req.constructors import ( - install_req_from_parsed_requirement - ) -except Exception: - from pip.download import PipSession - from pip.req.req_file import parse_requirements - - def install_req_from_parsed_requirement(req): - return req - -from collections import defaultdict +from odoo_tools.compat import Path +from odoo_tools.modules.search import find_addons_paths, find_modules_paths +from odoo_tools.configuration.pips import merge_requirements try: # python3 @@ -68,63 +53,6 @@ def quote(s): return "'" + s.replace("'", "'\"'\"'") + "'" -class Requirement(object): - def __init__(self): - self.extras = set() - self.specifiers = set() - self.links = set() - self.editable = False - - -def merge_requirements(files): - requirements = defaultdict(lambda: Requirement()) - links = set() - - for filename in files: - f_requirements = parse_requirements( - str(filename), session=PipSession() - ) - for parsed_requirement in f_requirements: - requirement = install_req_from_parsed_requirement( - parsed_requirement - ) - - if requirement.markers and not requirement.markers.evaluate(): - continue - - if not hasattr(requirement.req, 'name'): - links.add(requirement.link.url) - break - name = requirement.req.name - specifiers = requirement.req.specifier - extras = requirement.req.extras - requirements[name].extras |= set(extras) - requirements[name].specifiers |= set(specifiers) - if requirement.link: - requirements[name].links |= {requirement.link.url} - requirements[name].editable |= requirement.editable - - result = [] - for key, value in requirements.items(): - if value.links: - result.append("%s" % value.links.pop()) - elif not value.extras: - result.append( - "%s %s" % (key, ",".join(map(str, value.specifiers))) - ) - else: - result.append("%s [%s] %s" % ( - key, - ",".join(map(str, value.extras)), - ",".join(map(str, value.specifiers)) - )) - - for link in links: - result.append(link) - - return "\n".join(result) - - def pipe(args): """ Call the process with std(in,out,err) @@ -178,42 +106,12 @@ def call_sudo_entrypoint(): return ret -def get_all_manifests(): - odoo_manifests = glob.glob('/addons/**/__manifest__.py', recursive=True) - erp_manifest = glob.glob('/addons/**/__openerp__.py', recursive=True) - - return odoo_manifests + erp_manifest - - -def module_name(manifest_file): - return path.basename(path.dirname(manifest_file)) - - -def get_module(manifest_file): - with open(manifest_file) as manifest_in: - try: - code = compile(manifest_in.read(), '__manifest__', 'eval') - module = eval(code, {}, {}) - except Exception: - module = {} - - return module - - -def is_installable(manifest): - return manifest.get('installable', True) - - -def is_server_wide(manifest): - return manifest.get('server_wide', False) - - def get_server_wide_modules(manifests): base_server_wide_modules = ['base', 'web'] custom_server_wide_modules = [ - module_name(filename) - for filename, module in manifests.items() - if is_server_wide(module) + manifest.technical_name + for manifest in manifests + if manifest.server_wide ] return base_server_wide_modules + custom_server_wide_modules @@ -246,7 +144,12 @@ def install_python_dependencies(valid_paths): req_file = Path('/var/lib/odoo/requirements.txt') with req_file.open('w') as fout: data = merge_requirements(requirement_files) - fout.write(data) + data = "\n".join(data) + + try: + fout.write(data) + except Exception: + fout.write(data.decode('utf-8')) for requirements in requirement_files: print("Installing python packages from %s" % requirements) @@ -338,65 +241,44 @@ def get_extra_paths(): ] -def get_valid_paths(): - base_addons = os.environ.get('ODOO_BASE_PATH') +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] - # addons = os.listdir('/addons') - valid_paths = [base_addons] +def get_valid_paths(): + base_addons = os.environ['ODOO_BASE_PATH'] - addons_paths = get_dirs('/addons') - addons_paths += get_extra_paths() + base_addons_paths = [base_addons, '/addons'] + base_addons_paths += get_extra_paths() - for addons_path in addons_paths: - print("Lookup addons in %s" % addons_path) - flush_streams() - addons = get_dirs(addons_path) - for addon in addons: - files = os.listdir(addon) - if ( - '__init__.py' in files and - '__manifest__.py' in files or - '__openerp__.py' in files - ): - valid_paths.append(addons_path) - print("Addons found !") - flush_streams() - break - if addons_path in valid_paths: - continue - else: - print("No addons found in path. Skipping...") - flush_streams() + valid_paths = find_addons_paths(base_addons_paths) + + for valid_path in valid_paths: + print("Found installable modules in {}".format(valid_path)) return valid_paths def setup_addons_paths(config, valid_paths): - config.set('options', 'addons_path', ",".join(valid_paths)) + config.set('options', 'addons_path', ",".join([ + str(path) for path in valid_paths + ])) flush_streams() -def setup_server_wide_modules(config): +def setup_server_wide_modules(config, valid_paths): print("Searching for server wide modules") - all_manifests = get_all_manifests() + filters = set(['installable']) + all_manifests = find_modules_paths(valid_paths, filters) - print("Found %s modules" % (len(all_manifests))) + print("Found %s installable modules " % (len(all_manifests),)) - manifests = { - filename: get_module(filename) - for filename in all_manifests - } - - installable_manifests = { - filename: module - for filename, module in manifests.items() - if is_installable(module) - } - - print("Found %s installable modules " % (len(installable_manifests),)) - - server_wide_modules = get_server_wide_modules(installable_manifests) + server_wide_modules = get_server_wide_modules(all_manifests) if len(server_wide_modules) > 2: modules = ",".join(server_wide_modules) @@ -408,6 +290,36 @@ def setup_server_wide_modules(config): flush_streams() +def convert_value(name, value): + if value in ['True', 'False']: + return value == 'True' + + return value + + +def setup_env_config(config): + try: + from odoo.tools import config as odoo_config + except ImportError: + from openerp.tools import config as odoo_config + + params_by_name = {} + for key, opt in odoo_config.casts.items(): + value_opt = opt.get_opt_string() + value_opt = value_opt.upper().replace('--', 'ODOO_') + value_opt = value_opt.replace('-', '_') + params_by_name[value_opt] = key + + for key, value in os.environ.items(): + if not key.startswith('ODOO_'): + continue + + if key in params_by_name: + config_name = params_by_name[key] + converted_value = convert_value(key, value) + config.set('options', config_name, converted_value) + + def setup_environ(config): print("Configuring environment variables for postgresql") @@ -499,12 +411,39 @@ def wait_postgresql(): @contextmanager def get_config(config_path): config = ConfigParser() - config.read(config_path) + config.read(str(config_path)) yield config - with config_path.open('w') as out: - config.write(out) + try: + with config_path.open('w') as out: + config.write(out) + except Exception: + with config_path.open('wb') as out: + config.write(out) + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for vpath in paths: + cur_path = Path(vpath) + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(vpath) + + return res_paths def main(): @@ -518,7 +457,9 @@ def main(): sys.exit(ret) # Install python packages with pip in user home - valid_paths = get_valid_paths() + orig_valid_paths = get_valid_paths() + excluded_paths = get_excluded_paths() + valid_paths = filter_valid_paths(orig_valid_paths, excluded_paths) config_path = Path(os.environ.get('ODOO_RC', '/etc/odoo/odoo.cfg')) @@ -528,7 +469,8 @@ def main(): install_master_password(config) setup_environ(config) setup_addons_paths(config, valid_paths) - setup_server_wide_modules(config) + setup_server_wide_modules(config, valid_paths) + setup_env_config(config) if not os.environ.get('ODOO_SKIP_POSTGRES_WAIT'): wait_postgresql() diff --git a/13.0-nightly/find_modules.py b/13.0-nightly/find_modules.py deleted file mode 100644 index 9cb943748..000000000 --- a/13.0-nightly/find_modules.py +++ /dev/null @@ -1,76 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -from itertools import chain - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - "-p", - dest='paths', - action="append", - help="Location in which to search", - default=[] - ) - - parser.add_option( - "--only-name", - dest="only_name", - action="store_true", - help="Only display module name instead of path", - default=False, - ) - - parser.add_option( - '--csv', - dest="is_csv", - action="store_true", - help="Output as a comma separated list", - default=False - ) - - return parser - - -def find_modules(options, path): - modules = set() - - path = Path.cwd() / path - - erp_manifest = '__openerp__.py' - odoo_manifest = '__manifest__.py' - - manifest_globs = chain( - path.glob('**/{}'.format(erp_manifest)), - path.glob('**/{}'.format(odoo_manifest)), - ) - - for path in manifest_globs: - rel_path = path.parent.relative_to(Path.cwd()) - if options.only_name: - modules.add(rel_path.name) - else: - modules.add(str(rel_path)) - - return modules - - -def main(options, args): - modules = set() - for path in options.paths: - modules = modules.union(find_modules(options, path)) - - return modules - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - modules = main(options, args) - - if not options.is_csv: - for module in modules: - print(module) - else: - print(",".join(modules), end="") diff --git a/13.0-nightly/platform.sh b/13.0-nightly/platform.sh new file mode 100644 index 000000000..fdf5e42eb --- /dev/null +++ b/13.0-nightly/platform.sh @@ -0,0 +1,13 @@ +PLATFORM=`uname -m` + +if [ $PLATFORM = "aarch64" ] +then + echo -n "arm64" +else +if [ $PLATFORM = "x86_64" ] +then + echo -n "amd64" +else + echo -n $PLATFORM +fi +fi diff --git a/13.0-nightly/prepare_project.py b/13.0-nightly/prepare_project.py deleted file mode 100644 index 74a53f2e0..000000000 --- a/13.0-nightly/prepare_project.py +++ /dev/null @@ -1,198 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -import toml -from giturlparse import parse -from contextlib import contextmanager -import os -from subprocess import run -from urllib.parse import urlparse - - -def get_services(services): - return { - service.get('name'): service - for service in services.get('services') - } - - -def addons_by_project(options, addons): - res = {} - - for addon in addons: - if addon['url'] == 'self': - if options.ignore_self: - continue - parsed = parse(options.url) - else: - parsed = parse(addon['url']) - - auth = parsed.protocol in ['git', 'ssh'] - - res[parsed.repo] = dict( - addon, - url=parsed.url2https, - auth=auth - ) - - return res - - -def merge_addons(options, base, other): - base_addons = addons_by_project(options, base) - other_addons = addons_by_project(options, other) - - for name, addon in other_addons.items(): - if name not in base_addons: - base_addons[name] = addon - else: - base_addons[name] = dict(base_addons[name], **addon) - - return [ - addon - for addon in base_addons.values() - ] - - -def merge_services(options, base, other): - basic_inherit = dict(base, **other) - - if base.get('addons') or other.get('addons'): - basic_inherit['addons'] = merge_addons( - options, - base.get('addons', []), - other.get('addons', []) - ) - - return basic_inherit - - -def compile_service(options, services, name): - service = services.get(name, {}) - if 'inherit' in service: - merge_service = compile_service(options, services, service['inherit']) - service = merge_services(options, merge_service, service) - - return service - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - '-f', - '--file', - dest="file", - help="Input File" - ) - - parser.add_option( - '--url', - dest='url', - help="Url of self project" - ) - - parser.add_option( - '-o', - '--output', - dest="output_directory", - help="Output Directory" - ) - - parser.add_option( - '-e', - dest="env", - help="Environment to prepare" - ) - - parser.add_option( - '--username', - dest="username", - help="Username to replace with", - ) - - parser.add_option( - '--password', - dest="password", - help="password to set on https urls" - ) - - parser.add_option( - '-b', - '--branch', - dest="branch", - help="Default branch if no ref is defined" - ) - - parser.add_option( - '--ignore-self', - dest="ignore_self", - action="store_true", - help="Ignore self url as it's already fetched", - default=False - ) - - return parser - - -@contextmanager -def cd(directory): - cwd = Path.cwd() - try: - os.chdir(directory) - yield - finally: - os.chdir(cwd) - - -def fetch_addons(options, addon): - parsed = parse(addon['url']) - - url = urlparse(parsed.url2https) - - if addon['auth'] and options.username and options.password: - url = url._replace( - netloc="{}:{}@{}".format( - options.username, - options.password, - url.netloc - ) - ) - - repo_path = Path.cwd() / options.output_directory / parsed.repo - - repo_path.mkdir(exist_ok=True) - - with cd(repo_path): - run(['git', 'init']) - run(['git', 'remote', 'add', 'origin', url.geturl()]) - - ref = addon.get('commit') or addon.get('branch') or options.branch - - if ref: - run(['git', 'fetch', 'origin', ref]) - else: - run(['git', 'fetch', 'origin']) - - run(['git', 'checkout', 'FETCH_HEAD']) - run(['git', 'remote', 'remove', 'origin']) - - -def main(options, args): - with Path(options.file).open('r') as fin: - services = toml.loads(fin.read()) - - by_name = get_services(services) - - outputdir = Path(options.output_directory) - outputdir.mkdir(exist_ok=True) - - service = compile_service(options, by_name, options.env) - - for addons in service.get('addons', []): - fetch_addons(options, addons) - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - main(options, args) diff --git a/13.0-nightly/sudo-entrypoint.py b/13.0-nightly/sudo-entrypoint.py index 98779baae..916239170 100755 --- a/13.0-nightly/sudo-entrypoint.py +++ b/13.0-nightly/sudo-entrypoint.py @@ -12,6 +12,7 @@ from os import path from os.path import expanduser import shutil +import six try: from pathlib import Path @@ -69,6 +70,37 @@ def get_addons_paths(): ] +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for cur_path in paths: + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(cur_path) + + return res_paths + + def install_apt_packages(): """ Install debian dependencies. @@ -76,26 +108,34 @@ def install_apt_packages(): package_list = set() paths = get_addons_paths() + excluded_paths = get_excluded_paths() + paths = filter_valid_paths(paths, excluded_paths) print("Looking up for packages in {}".format(paths)) for addons_path in paths: for packages in addons_path.glob('**/apt-packages.txt'): print("Installing packages from %s" % packages) - with open(packages, 'r') as pack_file: - lines = [line.strip() for line in pack_file] + with packages.open('r') as pack_file: + lines = [ + six.ensure_text(line).strip() + for line in pack_file + if six.ensure_text(line).strip() + ] package_list.update(set(lines)) extras = os.environ.get('EXTRA_APT_PACKAGES', '') - print(f"Adding extra packages {extras}") + print("Adding extra packages {extras}".format(extras=extras)) if extras: for package in extras.split(','): - if not package: + if not package.strip(): continue - package_list.add(package) + package_list.add( + six.ensure_text(package).strip() + ) if len(package_list) > 0: - print(f"Installing {package_list}") + print("Installing {package_list}".format(package_list=package_list)) ret = pipe(['apt-get', 'update']) # Something went wrong, stop the service as it's failing diff --git a/13.0/Dockerfile b/13.0/Dockerfile index 4c5ea83d4..bef214edc 100644 --- a/13.0/Dockerfile +++ b/13.0/Dockerfile @@ -5,89 +5,93 @@ LABEL maintainer="Archeti " ENV LANG C.UTF-8 ARG DEBIAN_FRONTEND=noninteractive +COPY ./platform.sh /platform.sh # Install some dependencies python3.7 RUN set -x; \ - apt-get update \ + export ARCH=`sh /platform.sh` \ + # Setup deb source for postgresql client + && apt-get update \ && apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg \ + python3.7 \ python3-wheel \ python3-setuptools \ python3-pip \ - # python3.7 \ - # libpython3.7 \ + python3-psycopg2 \ + python3-ldap \ + python3-libsass \ + python3-lxml \ + python3-pillow \ + python3-pypdf2 \ + python3-psutil \ + python3-asn1crypto \ curl \ - gnupg \ - libpq-dev \ - libsasl2-2 \ - libldap-2.4-2 \ - libxml2 \ - libxmlsec1 \ - libxslt1.1 \ + # libpq-dev \ + # libsasl2-2 \ + # libldap-2.4-2 \ + # libxml2 \ + # libxmlsec1 \ + # libxslt1.1 \ sudo \ node-less \ - # python3-yaml \ - && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 \ - # && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 2 \ - && update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1 \ + && echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ + && export GNUPGHOME="$(mktemp -d)" \ + && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ + && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ + && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ + && gpgconf --kill all \ + && rm -rf "$GNUPGHOME" \ + # Download deb for wkhtmltox + && echo "Downloading https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb" \ + && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + ./wkhtmltox.deb \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 \ && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 2 \ - && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb \ - && apt-get install -y --no-install-recommends ./wkhtmltox.deb \ && rm -rf /var/lib/apt/lists/* wkhtmltox.deb -# Install latest postgresql-client -RUN set -x; \ - echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ - && export GNUPGHOME="$(mktemp -d)" \ - && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ - && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ - && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ - && gpgconf --kill all \ - && rm -rf "$GNUPGHOME" \ - && apt-get update \ - && apt-get install -y postgresql-client \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH=/usr/local/bin:$PATH +ENV PATH=/var/lib/odoo/.local/bin:/usr/local/bin:$PATH # Install Odoo Including things from sources ENV ODOO_VERSION 13.0 -ENV ODOO_RELEASE=20200623 +ENV ODOO_RELEASE=20221104 ARG ODOO_ARCHIVE=odoo_${ODOO_VERSION}.${ODOO_RELEASE}.tar.gz RUN set -x; \ apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ + # build-essential \ + # python3.7-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ && pip install -U pip \ && /usr/bin/env pip install \ - psycogreen \ + odoo-tools \ \ - && curl -o requirements.txt https://raw.githubusercontent.com/odoo/odoo/${ODOO_VERSION}/requirements.txt \ - && pip install -r requirements.txt \ - && /usr/bin/env pip install https://nightly.odoo.com/${ODOO_VERSION}/nightly/src/${ODOO_ARCHIVE} \ + && odootools manage setup --release "20221104" --repo "https://github.com/odoo/odoo.git" --ref "" "13.0" \ && cd / \ && apt-get --purge remove -y \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ + # build-essential \ + # python3.7-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* - -VOLUME /etc/odoo -VOLUME /var/lib/odoo + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /root/.cache -COPY ./odoo.conf /etc/odoo/ -COPY ./entrypoint.py / -COPY ./sudo-entrypoint.py / -COPY ./find_modules.py /scripts/find_modules.py -COPY ./prepare_project.py /scripts/prepare_project.py +# COPY ./odoo.conf /etc/odoo/ +# COPY ./entrypoint.py / +# COPY ./sudo-entrypoint.py / ARG UID=1000 ARG GID=1000 @@ -95,12 +99,15 @@ ARG GID=1000 RUN mkdir /addons \ && groupadd -r -g ${GID} odoo \ && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ - && chown odoo /etc/odoo/odoo.conf \ + # && chown odoo /etc/odoo/odoo.conf \ && chown -R odoo:odoo /addons \ - && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /sudo-entrypoint.py" >> /etc/sudoers + && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /usr/local/bin/odootools" >> /etc/sudoers + +# VOLUME /etc/odoo +VOLUME /var/lib/odoo -ENV ODOO_RC /etc/odoo/odoo.conf -ENV ODOO_BASE_PATH /usr/local/lib/python3.6/dist-packages/odoo/addons +ENV ODOO_RC /var/lib/odoo/odoo.conf +ENV ODOO_BASE_PATH /usr/local/lib/python3.7/dist-packages/odoo/addons # Env variable defined to monitor the kind of service running # it could be a staging/production/test or anything and undefined # is the default in case we need to know servers that aren't correctly @@ -113,19 +120,19 @@ expose 8071 USER odoo LABEL version="13.0" -LABEL release="20200623" +LABEL release="20221104" -LABEL org.opencontainers.image.created="2021-06-25T17:55:41.196719" +LABEL org.opencontainers.image.created="2022-11-04T20:06:13.803285" LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" -LABEL org.opencontainers.image.authors="Archeti " +LABEL org.opencontainers.image.authors="OdooPlus " LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.version="13.0" -LABEL org.opencontainers.image.vendor="ArcheTI" +LABEL org.opencontainers.image.vendor="OdooPlus" LABEL org.opencontainers.image.ref.name="13.0" LABEL org.opencontainers.image.title="Odoo 13.0" LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." -ENTRYPOINT ["/entrypoint.py"] +ENTRYPOINT ["odootools", "--log-level", "INFO", "entrypoint", "user"] cmd ["odoo"] diff --git a/13.0/entrypoint.py b/13.0/entrypoint.py index f2ce4654b..bb0ffddc3 100755 --- a/13.0/entrypoint.py +++ b/13.0/entrypoint.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # import argparse +from __future__ import print_function import time import shlex import subprocess @@ -17,25 +18,9 @@ from passlib.context import CryptContext from contextlib import contextmanager -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - -try: - from pip._internal.network.session import PipSession - from pip._internal.req.req_file import parse_requirements - from pip._internal.req.constructors import ( - install_req_from_parsed_requirement - ) -except Exception: - from pip.download import PipSession - from pip.req.req_file import parse_requirements - - def install_req_from_parsed_requirement(req): - return req - -from collections import defaultdict +from odoo_tools.compat import Path +from odoo_tools.modules.search import find_addons_paths, find_modules_paths +from odoo_tools.configuration.pips import merge_requirements try: # python3 @@ -68,63 +53,6 @@ def quote(s): return "'" + s.replace("'", "'\"'\"'") + "'" -class Requirement(object): - def __init__(self): - self.extras = set() - self.specifiers = set() - self.links = set() - self.editable = False - - -def merge_requirements(files): - requirements = defaultdict(lambda: Requirement()) - links = set() - - for filename in files: - f_requirements = parse_requirements( - str(filename), session=PipSession() - ) - for parsed_requirement in f_requirements: - requirement = install_req_from_parsed_requirement( - parsed_requirement - ) - - if requirement.markers and not requirement.markers.evaluate(): - continue - - if not hasattr(requirement.req, 'name'): - links.add(requirement.link.url) - break - name = requirement.req.name - specifiers = requirement.req.specifier - extras = requirement.req.extras - requirements[name].extras |= set(extras) - requirements[name].specifiers |= set(specifiers) - if requirement.link: - requirements[name].links |= {requirement.link.url} - requirements[name].editable |= requirement.editable - - result = [] - for key, value in requirements.items(): - if value.links: - result.append("%s" % value.links.pop()) - elif not value.extras: - result.append( - "%s %s" % (key, ",".join(map(str, value.specifiers))) - ) - else: - result.append("%s [%s] %s" % ( - key, - ",".join(map(str, value.extras)), - ",".join(map(str, value.specifiers)) - )) - - for link in links: - result.append(link) - - return "\n".join(result) - - def pipe(args): """ Call the process with std(in,out,err) @@ -178,42 +106,12 @@ def call_sudo_entrypoint(): return ret -def get_all_manifests(): - odoo_manifests = glob.glob('/addons/**/__manifest__.py', recursive=True) - erp_manifest = glob.glob('/addons/**/__openerp__.py', recursive=True) - - return odoo_manifests + erp_manifest - - -def module_name(manifest_file): - return path.basename(path.dirname(manifest_file)) - - -def get_module(manifest_file): - with open(manifest_file) as manifest_in: - try: - code = compile(manifest_in.read(), '__manifest__', 'eval') - module = eval(code, {}, {}) - except Exception: - module = {} - - return module - - -def is_installable(manifest): - return manifest.get('installable', True) - - -def is_server_wide(manifest): - return manifest.get('server_wide', False) - - def get_server_wide_modules(manifests): base_server_wide_modules = ['base', 'web'] custom_server_wide_modules = [ - module_name(filename) - for filename, module in manifests.items() - if is_server_wide(module) + manifest.technical_name + for manifest in manifests + if manifest.server_wide ] return base_server_wide_modules + custom_server_wide_modules @@ -246,7 +144,12 @@ def install_python_dependencies(valid_paths): req_file = Path('/var/lib/odoo/requirements.txt') with req_file.open('w') as fout: data = merge_requirements(requirement_files) - fout.write(data) + data = "\n".join(data) + + try: + fout.write(data) + except Exception: + fout.write(data.decode('utf-8')) for requirements in requirement_files: print("Installing python packages from %s" % requirements) @@ -338,65 +241,44 @@ def get_extra_paths(): ] -def get_valid_paths(): - base_addons = os.environ.get('ODOO_BASE_PATH') +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] - # addons = os.listdir('/addons') - valid_paths = [base_addons] +def get_valid_paths(): + base_addons = os.environ['ODOO_BASE_PATH'] - addons_paths = get_dirs('/addons') - addons_paths += get_extra_paths() + base_addons_paths = [base_addons, '/addons'] + base_addons_paths += get_extra_paths() - for addons_path in addons_paths: - print("Lookup addons in %s" % addons_path) - flush_streams() - addons = get_dirs(addons_path) - for addon in addons: - files = os.listdir(addon) - if ( - '__init__.py' in files and - '__manifest__.py' in files or - '__openerp__.py' in files - ): - valid_paths.append(addons_path) - print("Addons found !") - flush_streams() - break - if addons_path in valid_paths: - continue - else: - print("No addons found in path. Skipping...") - flush_streams() + valid_paths = find_addons_paths(base_addons_paths) + + for valid_path in valid_paths: + print("Found installable modules in {}".format(valid_path)) return valid_paths def setup_addons_paths(config, valid_paths): - config.set('options', 'addons_path', ",".join(valid_paths)) + config.set('options', 'addons_path', ",".join([ + str(path) for path in valid_paths + ])) flush_streams() -def setup_server_wide_modules(config): +def setup_server_wide_modules(config, valid_paths): print("Searching for server wide modules") - all_manifests = get_all_manifests() + filters = set(['installable']) + all_manifests = find_modules_paths(valid_paths, filters) - print("Found %s modules" % (len(all_manifests))) + print("Found %s installable modules " % (len(all_manifests),)) - manifests = { - filename: get_module(filename) - for filename in all_manifests - } - - installable_manifests = { - filename: module - for filename, module in manifests.items() - if is_installable(module) - } - - print("Found %s installable modules " % (len(installable_manifests),)) - - server_wide_modules = get_server_wide_modules(installable_manifests) + server_wide_modules = get_server_wide_modules(all_manifests) if len(server_wide_modules) > 2: modules = ",".join(server_wide_modules) @@ -408,6 +290,36 @@ def setup_server_wide_modules(config): flush_streams() +def convert_value(name, value): + if value in ['True', 'False']: + return value == 'True' + + return value + + +def setup_env_config(config): + try: + from odoo.tools import config as odoo_config + except ImportError: + from openerp.tools import config as odoo_config + + params_by_name = {} + for key, opt in odoo_config.casts.items(): + value_opt = opt.get_opt_string() + value_opt = value_opt.upper().replace('--', 'ODOO_') + value_opt = value_opt.replace('-', '_') + params_by_name[value_opt] = key + + for key, value in os.environ.items(): + if not key.startswith('ODOO_'): + continue + + if key in params_by_name: + config_name = params_by_name[key] + converted_value = convert_value(key, value) + config.set('options', config_name, converted_value) + + def setup_environ(config): print("Configuring environment variables for postgresql") @@ -499,12 +411,39 @@ def wait_postgresql(): @contextmanager def get_config(config_path): config = ConfigParser() - config.read(config_path) + config.read(str(config_path)) yield config - with config_path.open('w') as out: - config.write(out) + try: + with config_path.open('w') as out: + config.write(out) + except Exception: + with config_path.open('wb') as out: + config.write(out) + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for vpath in paths: + cur_path = Path(vpath) + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(vpath) + + return res_paths def main(): @@ -518,7 +457,9 @@ def main(): sys.exit(ret) # Install python packages with pip in user home - valid_paths = get_valid_paths() + orig_valid_paths = get_valid_paths() + excluded_paths = get_excluded_paths() + valid_paths = filter_valid_paths(orig_valid_paths, excluded_paths) config_path = Path(os.environ.get('ODOO_RC', '/etc/odoo/odoo.cfg')) @@ -528,7 +469,8 @@ def main(): install_master_password(config) setup_environ(config) setup_addons_paths(config, valid_paths) - setup_server_wide_modules(config) + setup_server_wide_modules(config, valid_paths) + setup_env_config(config) if not os.environ.get('ODOO_SKIP_POSTGRES_WAIT'): wait_postgresql() diff --git a/13.0/find_modules.py b/13.0/find_modules.py deleted file mode 100644 index 9cb943748..000000000 --- a/13.0/find_modules.py +++ /dev/null @@ -1,76 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -from itertools import chain - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - "-p", - dest='paths', - action="append", - help="Location in which to search", - default=[] - ) - - parser.add_option( - "--only-name", - dest="only_name", - action="store_true", - help="Only display module name instead of path", - default=False, - ) - - parser.add_option( - '--csv', - dest="is_csv", - action="store_true", - help="Output as a comma separated list", - default=False - ) - - return parser - - -def find_modules(options, path): - modules = set() - - path = Path.cwd() / path - - erp_manifest = '__openerp__.py' - odoo_manifest = '__manifest__.py' - - manifest_globs = chain( - path.glob('**/{}'.format(erp_manifest)), - path.glob('**/{}'.format(odoo_manifest)), - ) - - for path in manifest_globs: - rel_path = path.parent.relative_to(Path.cwd()) - if options.only_name: - modules.add(rel_path.name) - else: - modules.add(str(rel_path)) - - return modules - - -def main(options, args): - modules = set() - for path in options.paths: - modules = modules.union(find_modules(options, path)) - - return modules - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - modules = main(options, args) - - if not options.is_csv: - for module in modules: - print(module) - else: - print(",".join(modules), end="") diff --git a/13.0/platform.sh b/13.0/platform.sh new file mode 100644 index 000000000..fdf5e42eb --- /dev/null +++ b/13.0/platform.sh @@ -0,0 +1,13 @@ +PLATFORM=`uname -m` + +if [ $PLATFORM = "aarch64" ] +then + echo -n "arm64" +else +if [ $PLATFORM = "x86_64" ] +then + echo -n "amd64" +else + echo -n $PLATFORM +fi +fi diff --git a/13.0/prepare_project.py b/13.0/prepare_project.py deleted file mode 100644 index 74a53f2e0..000000000 --- a/13.0/prepare_project.py +++ /dev/null @@ -1,198 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -import toml -from giturlparse import parse -from contextlib import contextmanager -import os -from subprocess import run -from urllib.parse import urlparse - - -def get_services(services): - return { - service.get('name'): service - for service in services.get('services') - } - - -def addons_by_project(options, addons): - res = {} - - for addon in addons: - if addon['url'] == 'self': - if options.ignore_self: - continue - parsed = parse(options.url) - else: - parsed = parse(addon['url']) - - auth = parsed.protocol in ['git', 'ssh'] - - res[parsed.repo] = dict( - addon, - url=parsed.url2https, - auth=auth - ) - - return res - - -def merge_addons(options, base, other): - base_addons = addons_by_project(options, base) - other_addons = addons_by_project(options, other) - - for name, addon in other_addons.items(): - if name not in base_addons: - base_addons[name] = addon - else: - base_addons[name] = dict(base_addons[name], **addon) - - return [ - addon - for addon in base_addons.values() - ] - - -def merge_services(options, base, other): - basic_inherit = dict(base, **other) - - if base.get('addons') or other.get('addons'): - basic_inherit['addons'] = merge_addons( - options, - base.get('addons', []), - other.get('addons', []) - ) - - return basic_inherit - - -def compile_service(options, services, name): - service = services.get(name, {}) - if 'inherit' in service: - merge_service = compile_service(options, services, service['inherit']) - service = merge_services(options, merge_service, service) - - return service - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - '-f', - '--file', - dest="file", - help="Input File" - ) - - parser.add_option( - '--url', - dest='url', - help="Url of self project" - ) - - parser.add_option( - '-o', - '--output', - dest="output_directory", - help="Output Directory" - ) - - parser.add_option( - '-e', - dest="env", - help="Environment to prepare" - ) - - parser.add_option( - '--username', - dest="username", - help="Username to replace with", - ) - - parser.add_option( - '--password', - dest="password", - help="password to set on https urls" - ) - - parser.add_option( - '-b', - '--branch', - dest="branch", - help="Default branch if no ref is defined" - ) - - parser.add_option( - '--ignore-self', - dest="ignore_self", - action="store_true", - help="Ignore self url as it's already fetched", - default=False - ) - - return parser - - -@contextmanager -def cd(directory): - cwd = Path.cwd() - try: - os.chdir(directory) - yield - finally: - os.chdir(cwd) - - -def fetch_addons(options, addon): - parsed = parse(addon['url']) - - url = urlparse(parsed.url2https) - - if addon['auth'] and options.username and options.password: - url = url._replace( - netloc="{}:{}@{}".format( - options.username, - options.password, - url.netloc - ) - ) - - repo_path = Path.cwd() / options.output_directory / parsed.repo - - repo_path.mkdir(exist_ok=True) - - with cd(repo_path): - run(['git', 'init']) - run(['git', 'remote', 'add', 'origin', url.geturl()]) - - ref = addon.get('commit') or addon.get('branch') or options.branch - - if ref: - run(['git', 'fetch', 'origin', ref]) - else: - run(['git', 'fetch', 'origin']) - - run(['git', 'checkout', 'FETCH_HEAD']) - run(['git', 'remote', 'remove', 'origin']) - - -def main(options, args): - with Path(options.file).open('r') as fin: - services = toml.loads(fin.read()) - - by_name = get_services(services) - - outputdir = Path(options.output_directory) - outputdir.mkdir(exist_ok=True) - - service = compile_service(options, by_name, options.env) - - for addons in service.get('addons', []): - fetch_addons(options, addons) - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - main(options, args) diff --git a/13.0/sudo-entrypoint.py b/13.0/sudo-entrypoint.py index 98779baae..916239170 100755 --- a/13.0/sudo-entrypoint.py +++ b/13.0/sudo-entrypoint.py @@ -12,6 +12,7 @@ from os import path from os.path import expanduser import shutil +import six try: from pathlib import Path @@ -69,6 +70,37 @@ def get_addons_paths(): ] +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for cur_path in paths: + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(cur_path) + + return res_paths + + def install_apt_packages(): """ Install debian dependencies. @@ -76,26 +108,34 @@ def install_apt_packages(): package_list = set() paths = get_addons_paths() + excluded_paths = get_excluded_paths() + paths = filter_valid_paths(paths, excluded_paths) print("Looking up for packages in {}".format(paths)) for addons_path in paths: for packages in addons_path.glob('**/apt-packages.txt'): print("Installing packages from %s" % packages) - with open(packages, 'r') as pack_file: - lines = [line.strip() for line in pack_file] + with packages.open('r') as pack_file: + lines = [ + six.ensure_text(line).strip() + for line in pack_file + if six.ensure_text(line).strip() + ] package_list.update(set(lines)) extras = os.environ.get('EXTRA_APT_PACKAGES', '') - print(f"Adding extra packages {extras}") + print("Adding extra packages {extras}".format(extras=extras)) if extras: for package in extras.split(','): - if not package: + if not package.strip(): continue - package_list.add(package) + package_list.add( + six.ensure_text(package).strip() + ) if len(package_list) > 0: - print(f"Installing {package_list}") + print("Installing {package_list}".format(package_list=package_list)) ret = pipe(['apt-get', 'update']) # Something went wrong, stop the service as it's failing diff --git a/14.0-nightly/Dockerfile b/14.0-nightly/Dockerfile index 9a4db79ee..6eb1d077a 100644 --- a/14.0-nightly/Dockerfile +++ b/14.0-nightly/Dockerfile @@ -1,93 +1,97 @@ -FROM ubuntu:bionic +FROM ubuntu:focal LABEL maintainer="Archeti " # Generate locale C.UTF-8 for postgres and general locale data ENV LANG C.UTF-8 ARG DEBIAN_FRONTEND=noninteractive +COPY ./platform.sh /platform.sh # Install some dependencies python3.7 RUN set -x; \ - apt-get update \ + export ARCH=`sh /platform.sh` \ + # Setup deb source for postgresql client + && apt-get update \ && apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg \ + python3.8 \ python3-wheel \ python3-setuptools \ python3-pip \ - # python3.7 \ - # libpython3.7 \ + python3-psycopg2 \ + python3-ldap \ + python3-libsass \ + python3-lxml \ + python3-pillow \ + python3-pypdf2 \ + python3-psutil \ + python3-asn1crypto \ curl \ - gnupg \ - libpq-dev \ - libsasl2-2 \ - libldap-2.4-2 \ - libxml2 \ - libxmlsec1 \ - libxslt1.1 \ + # libpq-dev \ + # libsasl2-2 \ + # libldap-2.4-2 \ + # libxml2 \ + # libxmlsec1 \ + # libxslt1.1 \ sudo \ node-less \ - # python3-yaml \ - && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 \ - # && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 2 \ - && update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1 \ + && echo 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' > etc/apt/sources.list.d/pgdg.list \ + && export GNUPGHOME="$(mktemp -d)" \ + && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ + && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ + && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ + && gpgconf --kill all \ + && rm -rf "$GNUPGHOME" \ + # Download deb for wkhtmltox + && echo "Downloading https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_${ARCH}.deb" \ + && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_${ARCH}.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + ./wkhtmltox.deb \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1 \ && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 2 \ - && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb \ - && apt-get install -y --no-install-recommends ./wkhtmltox.deb \ && rm -rf /var/lib/apt/lists/* wkhtmltox.deb -# Install latest postgresql-client -RUN set -x; \ - echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ - && export GNUPGHOME="$(mktemp -d)" \ - && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ - && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ - && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ - && gpgconf --kill all \ - && rm -rf "$GNUPGHOME" \ - && apt-get update \ - && apt-get install -y postgresql-client \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH=/usr/local/bin:$PATH +ENV PATH=/var/lib/odoo/.local/bin:/usr/local/bin:$PATH # Install Odoo Including things from sources ENV ODOO_VERSION 14.0 -ENV ODOO_RELEASE=20210625 +ENV ODOO_RELEASE= ARG ODOO_ARCHIVE=odoo_${ODOO_VERSION}.${ODOO_RELEASE}.tar.gz RUN set -x; \ apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ + # build-essential \ + # python3.8-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ && pip install -U pip \ && /usr/bin/env pip install \ - psycogreen \ + odoo-tools \ \ - && curl -o requirements.txt https://raw.githubusercontent.com/odoo/odoo/${ODOO_VERSION}/requirements.txt \ - && pip install -r requirements.txt \ - && /usr/bin/env pip install https://nightly.odoo.com/${ODOO_VERSION}/nightly/src/${ODOO_ARCHIVE} \ + && odootools manage setup --release "" --repo "https://github.com/odoo/odoo.git" --ref "" "14.0" \ && cd / \ && apt-get --purge remove -y \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ + # build-essential \ + # python3.8-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* - -VOLUME /etc/odoo -VOLUME /var/lib/odoo + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /root/.cache -COPY ./odoo.conf /etc/odoo/ -COPY ./entrypoint.py / -COPY ./sudo-entrypoint.py / -COPY ./find_modules.py /scripts/find_modules.py -COPY ./prepare_project.py /scripts/prepare_project.py +# COPY ./odoo.conf /etc/odoo/ +# COPY ./entrypoint.py / +# COPY ./sudo-entrypoint.py / ARG UID=1000 ARG GID=1000 @@ -95,12 +99,15 @@ ARG GID=1000 RUN mkdir /addons \ && groupadd -r -g ${GID} odoo \ && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ - && chown odoo /etc/odoo/odoo.conf \ + # && chown odoo /etc/odoo/odoo.conf \ && chown -R odoo:odoo /addons \ - && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /sudo-entrypoint.py" >> /etc/sudoers + && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /usr/local/bin/odootools" >> /etc/sudoers + +# VOLUME /etc/odoo +VOLUME /var/lib/odoo -ENV ODOO_RC /etc/odoo/odoo.conf -ENV ODOO_BASE_PATH /usr/local/lib/python3.6/dist-packages/odoo/addons +ENV ODOO_RC /var/lib/odoo/odoo.conf +ENV ODOO_BASE_PATH /usr/local/lib/python3.8/dist-packages/odoo/addons # Env variable defined to monitor the kind of service running # it could be a staging/production/test or anything and undefined # is the default in case we need to know servers that aren't correctly @@ -113,19 +120,19 @@ expose 8071 USER odoo LABEL version="14.0" -LABEL release="20210625" +LABEL release="" -LABEL org.opencontainers.image.created="2021-06-25T17:55:41.202241" +LABEL org.opencontainers.image.created="2022-11-04T20:06:13.825010" LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" -LABEL org.opencontainers.image.authors="Archeti " +LABEL org.opencontainers.image.authors="OdooPlus " LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.version="14.0" -LABEL org.opencontainers.image.vendor="ArcheTI" +LABEL org.opencontainers.image.vendor="OdooPlus" LABEL org.opencontainers.image.ref.name="14.0-nightly" LABEL org.opencontainers.image.title="Odoo 14.0" LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." -ENTRYPOINT ["/entrypoint.py"] +ENTRYPOINT ["odootools", "--log-level", "INFO", "entrypoint", "user"] cmd ["odoo"] diff --git a/14.0-nightly/entrypoint.py b/14.0-nightly/entrypoint.py index f2ce4654b..bb0ffddc3 100755 --- a/14.0-nightly/entrypoint.py +++ b/14.0-nightly/entrypoint.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # import argparse +from __future__ import print_function import time import shlex import subprocess @@ -17,25 +18,9 @@ from passlib.context import CryptContext from contextlib import contextmanager -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - -try: - from pip._internal.network.session import PipSession - from pip._internal.req.req_file import parse_requirements - from pip._internal.req.constructors import ( - install_req_from_parsed_requirement - ) -except Exception: - from pip.download import PipSession - from pip.req.req_file import parse_requirements - - def install_req_from_parsed_requirement(req): - return req - -from collections import defaultdict +from odoo_tools.compat import Path +from odoo_tools.modules.search import find_addons_paths, find_modules_paths +from odoo_tools.configuration.pips import merge_requirements try: # python3 @@ -68,63 +53,6 @@ def quote(s): return "'" + s.replace("'", "'\"'\"'") + "'" -class Requirement(object): - def __init__(self): - self.extras = set() - self.specifiers = set() - self.links = set() - self.editable = False - - -def merge_requirements(files): - requirements = defaultdict(lambda: Requirement()) - links = set() - - for filename in files: - f_requirements = parse_requirements( - str(filename), session=PipSession() - ) - for parsed_requirement in f_requirements: - requirement = install_req_from_parsed_requirement( - parsed_requirement - ) - - if requirement.markers and not requirement.markers.evaluate(): - continue - - if not hasattr(requirement.req, 'name'): - links.add(requirement.link.url) - break - name = requirement.req.name - specifiers = requirement.req.specifier - extras = requirement.req.extras - requirements[name].extras |= set(extras) - requirements[name].specifiers |= set(specifiers) - if requirement.link: - requirements[name].links |= {requirement.link.url} - requirements[name].editable |= requirement.editable - - result = [] - for key, value in requirements.items(): - if value.links: - result.append("%s" % value.links.pop()) - elif not value.extras: - result.append( - "%s %s" % (key, ",".join(map(str, value.specifiers))) - ) - else: - result.append("%s [%s] %s" % ( - key, - ",".join(map(str, value.extras)), - ",".join(map(str, value.specifiers)) - )) - - for link in links: - result.append(link) - - return "\n".join(result) - - def pipe(args): """ Call the process with std(in,out,err) @@ -178,42 +106,12 @@ def call_sudo_entrypoint(): return ret -def get_all_manifests(): - odoo_manifests = glob.glob('/addons/**/__manifest__.py', recursive=True) - erp_manifest = glob.glob('/addons/**/__openerp__.py', recursive=True) - - return odoo_manifests + erp_manifest - - -def module_name(manifest_file): - return path.basename(path.dirname(manifest_file)) - - -def get_module(manifest_file): - with open(manifest_file) as manifest_in: - try: - code = compile(manifest_in.read(), '__manifest__', 'eval') - module = eval(code, {}, {}) - except Exception: - module = {} - - return module - - -def is_installable(manifest): - return manifest.get('installable', True) - - -def is_server_wide(manifest): - return manifest.get('server_wide', False) - - def get_server_wide_modules(manifests): base_server_wide_modules = ['base', 'web'] custom_server_wide_modules = [ - module_name(filename) - for filename, module in manifests.items() - if is_server_wide(module) + manifest.technical_name + for manifest in manifests + if manifest.server_wide ] return base_server_wide_modules + custom_server_wide_modules @@ -246,7 +144,12 @@ def install_python_dependencies(valid_paths): req_file = Path('/var/lib/odoo/requirements.txt') with req_file.open('w') as fout: data = merge_requirements(requirement_files) - fout.write(data) + data = "\n".join(data) + + try: + fout.write(data) + except Exception: + fout.write(data.decode('utf-8')) for requirements in requirement_files: print("Installing python packages from %s" % requirements) @@ -338,65 +241,44 @@ def get_extra_paths(): ] -def get_valid_paths(): - base_addons = os.environ.get('ODOO_BASE_PATH') +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] - # addons = os.listdir('/addons') - valid_paths = [base_addons] +def get_valid_paths(): + base_addons = os.environ['ODOO_BASE_PATH'] - addons_paths = get_dirs('/addons') - addons_paths += get_extra_paths() + base_addons_paths = [base_addons, '/addons'] + base_addons_paths += get_extra_paths() - for addons_path in addons_paths: - print("Lookup addons in %s" % addons_path) - flush_streams() - addons = get_dirs(addons_path) - for addon in addons: - files = os.listdir(addon) - if ( - '__init__.py' in files and - '__manifest__.py' in files or - '__openerp__.py' in files - ): - valid_paths.append(addons_path) - print("Addons found !") - flush_streams() - break - if addons_path in valid_paths: - continue - else: - print("No addons found in path. Skipping...") - flush_streams() + valid_paths = find_addons_paths(base_addons_paths) + + for valid_path in valid_paths: + print("Found installable modules in {}".format(valid_path)) return valid_paths def setup_addons_paths(config, valid_paths): - config.set('options', 'addons_path', ",".join(valid_paths)) + config.set('options', 'addons_path', ",".join([ + str(path) for path in valid_paths + ])) flush_streams() -def setup_server_wide_modules(config): +def setup_server_wide_modules(config, valid_paths): print("Searching for server wide modules") - all_manifests = get_all_manifests() + filters = set(['installable']) + all_manifests = find_modules_paths(valid_paths, filters) - print("Found %s modules" % (len(all_manifests))) + print("Found %s installable modules " % (len(all_manifests),)) - manifests = { - filename: get_module(filename) - for filename in all_manifests - } - - installable_manifests = { - filename: module - for filename, module in manifests.items() - if is_installable(module) - } - - print("Found %s installable modules " % (len(installable_manifests),)) - - server_wide_modules = get_server_wide_modules(installable_manifests) + server_wide_modules = get_server_wide_modules(all_manifests) if len(server_wide_modules) > 2: modules = ",".join(server_wide_modules) @@ -408,6 +290,36 @@ def setup_server_wide_modules(config): flush_streams() +def convert_value(name, value): + if value in ['True', 'False']: + return value == 'True' + + return value + + +def setup_env_config(config): + try: + from odoo.tools import config as odoo_config + except ImportError: + from openerp.tools import config as odoo_config + + params_by_name = {} + for key, opt in odoo_config.casts.items(): + value_opt = opt.get_opt_string() + value_opt = value_opt.upper().replace('--', 'ODOO_') + value_opt = value_opt.replace('-', '_') + params_by_name[value_opt] = key + + for key, value in os.environ.items(): + if not key.startswith('ODOO_'): + continue + + if key in params_by_name: + config_name = params_by_name[key] + converted_value = convert_value(key, value) + config.set('options', config_name, converted_value) + + def setup_environ(config): print("Configuring environment variables for postgresql") @@ -499,12 +411,39 @@ def wait_postgresql(): @contextmanager def get_config(config_path): config = ConfigParser() - config.read(config_path) + config.read(str(config_path)) yield config - with config_path.open('w') as out: - config.write(out) + try: + with config_path.open('w') as out: + config.write(out) + except Exception: + with config_path.open('wb') as out: + config.write(out) + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for vpath in paths: + cur_path = Path(vpath) + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(vpath) + + return res_paths def main(): @@ -518,7 +457,9 @@ def main(): sys.exit(ret) # Install python packages with pip in user home - valid_paths = get_valid_paths() + orig_valid_paths = get_valid_paths() + excluded_paths = get_excluded_paths() + valid_paths = filter_valid_paths(orig_valid_paths, excluded_paths) config_path = Path(os.environ.get('ODOO_RC', '/etc/odoo/odoo.cfg')) @@ -528,7 +469,8 @@ def main(): install_master_password(config) setup_environ(config) setup_addons_paths(config, valid_paths) - setup_server_wide_modules(config) + setup_server_wide_modules(config, valid_paths) + setup_env_config(config) if not os.environ.get('ODOO_SKIP_POSTGRES_WAIT'): wait_postgresql() diff --git a/14.0-nightly/find_modules.py b/14.0-nightly/find_modules.py deleted file mode 100644 index 9cb943748..000000000 --- a/14.0-nightly/find_modules.py +++ /dev/null @@ -1,76 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -from itertools import chain - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - "-p", - dest='paths', - action="append", - help="Location in which to search", - default=[] - ) - - parser.add_option( - "--only-name", - dest="only_name", - action="store_true", - help="Only display module name instead of path", - default=False, - ) - - parser.add_option( - '--csv', - dest="is_csv", - action="store_true", - help="Output as a comma separated list", - default=False - ) - - return parser - - -def find_modules(options, path): - modules = set() - - path = Path.cwd() / path - - erp_manifest = '__openerp__.py' - odoo_manifest = '__manifest__.py' - - manifest_globs = chain( - path.glob('**/{}'.format(erp_manifest)), - path.glob('**/{}'.format(odoo_manifest)), - ) - - for path in manifest_globs: - rel_path = path.parent.relative_to(Path.cwd()) - if options.only_name: - modules.add(rel_path.name) - else: - modules.add(str(rel_path)) - - return modules - - -def main(options, args): - modules = set() - for path in options.paths: - modules = modules.union(find_modules(options, path)) - - return modules - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - modules = main(options, args) - - if not options.is_csv: - for module in modules: - print(module) - else: - print(",".join(modules), end="") diff --git a/14.0-nightly/platform.sh b/14.0-nightly/platform.sh new file mode 100644 index 000000000..fdf5e42eb --- /dev/null +++ b/14.0-nightly/platform.sh @@ -0,0 +1,13 @@ +PLATFORM=`uname -m` + +if [ $PLATFORM = "aarch64" ] +then + echo -n "arm64" +else +if [ $PLATFORM = "x86_64" ] +then + echo -n "amd64" +else + echo -n $PLATFORM +fi +fi diff --git a/14.0-nightly/prepare_project.py b/14.0-nightly/prepare_project.py deleted file mode 100644 index 74a53f2e0..000000000 --- a/14.0-nightly/prepare_project.py +++ /dev/null @@ -1,198 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -import toml -from giturlparse import parse -from contextlib import contextmanager -import os -from subprocess import run -from urllib.parse import urlparse - - -def get_services(services): - return { - service.get('name'): service - for service in services.get('services') - } - - -def addons_by_project(options, addons): - res = {} - - for addon in addons: - if addon['url'] == 'self': - if options.ignore_self: - continue - parsed = parse(options.url) - else: - parsed = parse(addon['url']) - - auth = parsed.protocol in ['git', 'ssh'] - - res[parsed.repo] = dict( - addon, - url=parsed.url2https, - auth=auth - ) - - return res - - -def merge_addons(options, base, other): - base_addons = addons_by_project(options, base) - other_addons = addons_by_project(options, other) - - for name, addon in other_addons.items(): - if name not in base_addons: - base_addons[name] = addon - else: - base_addons[name] = dict(base_addons[name], **addon) - - return [ - addon - for addon in base_addons.values() - ] - - -def merge_services(options, base, other): - basic_inherit = dict(base, **other) - - if base.get('addons') or other.get('addons'): - basic_inherit['addons'] = merge_addons( - options, - base.get('addons', []), - other.get('addons', []) - ) - - return basic_inherit - - -def compile_service(options, services, name): - service = services.get(name, {}) - if 'inherit' in service: - merge_service = compile_service(options, services, service['inherit']) - service = merge_services(options, merge_service, service) - - return service - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - '-f', - '--file', - dest="file", - help="Input File" - ) - - parser.add_option( - '--url', - dest='url', - help="Url of self project" - ) - - parser.add_option( - '-o', - '--output', - dest="output_directory", - help="Output Directory" - ) - - parser.add_option( - '-e', - dest="env", - help="Environment to prepare" - ) - - parser.add_option( - '--username', - dest="username", - help="Username to replace with", - ) - - parser.add_option( - '--password', - dest="password", - help="password to set on https urls" - ) - - parser.add_option( - '-b', - '--branch', - dest="branch", - help="Default branch if no ref is defined" - ) - - parser.add_option( - '--ignore-self', - dest="ignore_self", - action="store_true", - help="Ignore self url as it's already fetched", - default=False - ) - - return parser - - -@contextmanager -def cd(directory): - cwd = Path.cwd() - try: - os.chdir(directory) - yield - finally: - os.chdir(cwd) - - -def fetch_addons(options, addon): - parsed = parse(addon['url']) - - url = urlparse(parsed.url2https) - - if addon['auth'] and options.username and options.password: - url = url._replace( - netloc="{}:{}@{}".format( - options.username, - options.password, - url.netloc - ) - ) - - repo_path = Path.cwd() / options.output_directory / parsed.repo - - repo_path.mkdir(exist_ok=True) - - with cd(repo_path): - run(['git', 'init']) - run(['git', 'remote', 'add', 'origin', url.geturl()]) - - ref = addon.get('commit') or addon.get('branch') or options.branch - - if ref: - run(['git', 'fetch', 'origin', ref]) - else: - run(['git', 'fetch', 'origin']) - - run(['git', 'checkout', 'FETCH_HEAD']) - run(['git', 'remote', 'remove', 'origin']) - - -def main(options, args): - with Path(options.file).open('r') as fin: - services = toml.loads(fin.read()) - - by_name = get_services(services) - - outputdir = Path(options.output_directory) - outputdir.mkdir(exist_ok=True) - - service = compile_service(options, by_name, options.env) - - for addons in service.get('addons', []): - fetch_addons(options, addons) - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - main(options, args) diff --git a/14.0-nightly/sudo-entrypoint.py b/14.0-nightly/sudo-entrypoint.py index 98779baae..916239170 100755 --- a/14.0-nightly/sudo-entrypoint.py +++ b/14.0-nightly/sudo-entrypoint.py @@ -12,6 +12,7 @@ from os import path from os.path import expanduser import shutil +import six try: from pathlib import Path @@ -69,6 +70,37 @@ def get_addons_paths(): ] +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for cur_path in paths: + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(cur_path) + + return res_paths + + def install_apt_packages(): """ Install debian dependencies. @@ -76,26 +108,34 @@ def install_apt_packages(): package_list = set() paths = get_addons_paths() + excluded_paths = get_excluded_paths() + paths = filter_valid_paths(paths, excluded_paths) print("Looking up for packages in {}".format(paths)) for addons_path in paths: for packages in addons_path.glob('**/apt-packages.txt'): print("Installing packages from %s" % packages) - with open(packages, 'r') as pack_file: - lines = [line.strip() for line in pack_file] + with packages.open('r') as pack_file: + lines = [ + six.ensure_text(line).strip() + for line in pack_file + if six.ensure_text(line).strip() + ] package_list.update(set(lines)) extras = os.environ.get('EXTRA_APT_PACKAGES', '') - print(f"Adding extra packages {extras}") + print("Adding extra packages {extras}".format(extras=extras)) if extras: for package in extras.split(','): - if not package: + if not package.strip(): continue - package_list.add(package) + package_list.add( + six.ensure_text(package).strip() + ) if len(package_list) > 0: - print(f"Installing {package_list}") + print("Installing {package_list}".format(package_list=package_list)) ret = pipe(['apt-get', 'update']) # Something went wrong, stop the service as it's failing diff --git a/14.0-tiny/Dockerfile b/14.0-tiny/Dockerfile deleted file mode 100644 index 72a1444f4..000000000 --- a/14.0-tiny/Dockerfile +++ /dev/null @@ -1,141 +0,0 @@ -FROM ubuntu:bionic -LABEL maintainer="Archeti " - -# Generate locale C.UTF-8 for postgres and general locale data -ENV LANG C.UTF-8 -ARG DEBIAN_FRONTEND=noninteractive - -# Install some dependencies python3.7 -RUN set -x; \ - apt-get update \ - && apt-get install -y --no-install-recommends \ - python3-wheel \ - python3-setuptools \ - python3-pip \ - # python3.7 \ - # libpython3.7 \ - curl \ - gnupg \ - libpq-dev \ - libsasl2-2 \ - libldap-2.4-2 \ - libxml2 \ - libxmlsec1 \ - libxslt1.1 \ - sudo \ - node-less \ - # python3-yaml \ - && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 \ - # && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 2 \ - && update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1 \ - && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 2 \ - && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb \ - && apt-get install -y --no-install-recommends ./wkhtmltox.deb \ - && rm -rf /var/lib/apt/lists/* wkhtmltox.deb - -# Install latest postgresql-client -RUN set -x; \ - echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ - && export GNUPGHOME="$(mktemp -d)" \ - && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ - && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ - && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ - && gpgconf --kill all \ - && rm -rf "$GNUPGHOME" \ - && apt-get update \ - && apt-get install -y postgresql-client \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH=/usr/local/bin:$PATH -# Install Odoo Including things from sources -ENV ODOO_VERSION 14.0 -ENV ODOO_RELEASE=20201009 -ARG ODOO_ARCHIVE=odoo_${ODOO_VERSION}.${ODOO_RELEASE}.tar.gz -RUN set -x; \ - apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ - && pip install -U pip \ - && /usr/bin/env pip install \ - psycogreen \ - \ - && curl -o requirements.txt https://raw.githubusercontent.com/odoo/odoo/${ODOO_VERSION}/requirements.txt \ - && pip install -r requirements.txt \ - && rm requirements.txt \ - && apt-get --purge remove -y \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* - - -COPY ./odoo.conf /etc/odoo/ -COPY ./entrypoint.py / -COPY ./sudo-entrypoint.py / -COPY ./find_modules.py /scripts/find_modules.py -COPY ./prepare_project.py /scripts/prepare_project.py - -ARG UID=1000 -ARG GID=1000 - -RUN mkdir /addons \ - && groupadd -r -g ${GID} odoo \ - && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ - && chown odoo /etc/odoo/odoo.conf \ - && chown -R odoo:odoo /addons \ - && chown -R odoo:odoo /var/lib/odoo \ - && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /sudo-entrypoint.py" >> /etc/sudoers - -USER odoo -WORKDIR /var/lib/odoo - -RUN set -x; \ - cd /var/lib/odoo \ - && pip install -U pip \ - && /usr/bin/env pip install https://nightly.odoo.com/${ODOO_VERSION}/nightly/src/${ODOO_ARCHIVE} \ - && cd .local/lib/python3.6/site-packages/odoo/addons \ - && find . -maxdepth 1 ! -regex "\(\.\/web\|\.\/base\|\.\|\.\.\)" -type d -exec rm -rf {} \; - - -VOLUME /etc/odoo -VOLUME /var/lib/odoo - -ENV ODOO_RC /etc/odoo/odoo.conf -ENV ODOO_BASE_PATH /usr/local/lib/python3.6/dist-packages/odoo/addons -# Env variable defined to monitor the kind of service running -# it could be a staging/production/test or anything and undefined -# is the default in case we need to know servers that aren't correctly -# defined -ENV DEPLOYMENT_AREA undefined - -expose 8069 -expose 8071 - -LABEL version="14.0" -LABEL release="20201009" - -LABEL org.opencontainers.image.created="2021-06-25T17:55:41.203156" -LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" -LABEL org.opencontainers.image.authors="Archeti " -LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" -LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" -LABEL org.opencontainers.image.version="14.0" -LABEL org.opencontainers.image.vendor="ArcheTI" -LABEL org.opencontainers.image.ref.name="14.0-tiny" -LABEL org.opencontainers.image.title="Odoo 14.0" -LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." - -ENTRYPOINT ["/entrypoint.py"] - -cmd ["odoo"] diff --git a/14.0-tiny/find_modules.py b/14.0-tiny/find_modules.py deleted file mode 100644 index 9cb943748..000000000 --- a/14.0-tiny/find_modules.py +++ /dev/null @@ -1,76 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -from itertools import chain - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - "-p", - dest='paths', - action="append", - help="Location in which to search", - default=[] - ) - - parser.add_option( - "--only-name", - dest="only_name", - action="store_true", - help="Only display module name instead of path", - default=False, - ) - - parser.add_option( - '--csv', - dest="is_csv", - action="store_true", - help="Output as a comma separated list", - default=False - ) - - return parser - - -def find_modules(options, path): - modules = set() - - path = Path.cwd() / path - - erp_manifest = '__openerp__.py' - odoo_manifest = '__manifest__.py' - - manifest_globs = chain( - path.glob('**/{}'.format(erp_manifest)), - path.glob('**/{}'.format(odoo_manifest)), - ) - - for path in manifest_globs: - rel_path = path.parent.relative_to(Path.cwd()) - if options.only_name: - modules.add(rel_path.name) - else: - modules.add(str(rel_path)) - - return modules - - -def main(options, args): - modules = set() - for path in options.paths: - modules = modules.union(find_modules(options, path)) - - return modules - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - modules = main(options, args) - - if not options.is_csv: - for module in modules: - print(module) - else: - print(",".join(modules), end="") diff --git a/14.0-tiny/prepare_project.py b/14.0-tiny/prepare_project.py deleted file mode 100644 index 74a53f2e0..000000000 --- a/14.0-tiny/prepare_project.py +++ /dev/null @@ -1,198 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -import toml -from giturlparse import parse -from contextlib import contextmanager -import os -from subprocess import run -from urllib.parse import urlparse - - -def get_services(services): - return { - service.get('name'): service - for service in services.get('services') - } - - -def addons_by_project(options, addons): - res = {} - - for addon in addons: - if addon['url'] == 'self': - if options.ignore_self: - continue - parsed = parse(options.url) - else: - parsed = parse(addon['url']) - - auth = parsed.protocol in ['git', 'ssh'] - - res[parsed.repo] = dict( - addon, - url=parsed.url2https, - auth=auth - ) - - return res - - -def merge_addons(options, base, other): - base_addons = addons_by_project(options, base) - other_addons = addons_by_project(options, other) - - for name, addon in other_addons.items(): - if name not in base_addons: - base_addons[name] = addon - else: - base_addons[name] = dict(base_addons[name], **addon) - - return [ - addon - for addon in base_addons.values() - ] - - -def merge_services(options, base, other): - basic_inherit = dict(base, **other) - - if base.get('addons') or other.get('addons'): - basic_inherit['addons'] = merge_addons( - options, - base.get('addons', []), - other.get('addons', []) - ) - - return basic_inherit - - -def compile_service(options, services, name): - service = services.get(name, {}) - if 'inherit' in service: - merge_service = compile_service(options, services, service['inherit']) - service = merge_services(options, merge_service, service) - - return service - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - '-f', - '--file', - dest="file", - help="Input File" - ) - - parser.add_option( - '--url', - dest='url', - help="Url of self project" - ) - - parser.add_option( - '-o', - '--output', - dest="output_directory", - help="Output Directory" - ) - - parser.add_option( - '-e', - dest="env", - help="Environment to prepare" - ) - - parser.add_option( - '--username', - dest="username", - help="Username to replace with", - ) - - parser.add_option( - '--password', - dest="password", - help="password to set on https urls" - ) - - parser.add_option( - '-b', - '--branch', - dest="branch", - help="Default branch if no ref is defined" - ) - - parser.add_option( - '--ignore-self', - dest="ignore_self", - action="store_true", - help="Ignore self url as it's already fetched", - default=False - ) - - return parser - - -@contextmanager -def cd(directory): - cwd = Path.cwd() - try: - os.chdir(directory) - yield - finally: - os.chdir(cwd) - - -def fetch_addons(options, addon): - parsed = parse(addon['url']) - - url = urlparse(parsed.url2https) - - if addon['auth'] and options.username and options.password: - url = url._replace( - netloc="{}:{}@{}".format( - options.username, - options.password, - url.netloc - ) - ) - - repo_path = Path.cwd() / options.output_directory / parsed.repo - - repo_path.mkdir(exist_ok=True) - - with cd(repo_path): - run(['git', 'init']) - run(['git', 'remote', 'add', 'origin', url.geturl()]) - - ref = addon.get('commit') or addon.get('branch') or options.branch - - if ref: - run(['git', 'fetch', 'origin', ref]) - else: - run(['git', 'fetch', 'origin']) - - run(['git', 'checkout', 'FETCH_HEAD']) - run(['git', 'remote', 'remove', 'origin']) - - -def main(options, args): - with Path(options.file).open('r') as fin: - services = toml.loads(fin.read()) - - by_name = get_services(services) - - outputdir = Path(options.output_directory) - outputdir.mkdir(exist_ok=True) - - service = compile_service(options, by_name, options.env) - - for addons in service.get('addons', []): - fetch_addons(options, addons) - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - main(options, args) diff --git a/14.0/Dockerfile b/14.0/Dockerfile index 7bb39a7d8..060290895 100644 --- a/14.0/Dockerfile +++ b/14.0/Dockerfile @@ -1,93 +1,97 @@ -FROM ubuntu:bionic +FROM ubuntu:focal LABEL maintainer="Archeti " # Generate locale C.UTF-8 for postgres and general locale data ENV LANG C.UTF-8 ARG DEBIAN_FRONTEND=noninteractive +COPY ./platform.sh /platform.sh # Install some dependencies python3.7 RUN set -x; \ - apt-get update \ + export ARCH=`sh /platform.sh` \ + # Setup deb source for postgresql client + && apt-get update \ && apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg \ + python3.8 \ python3-wheel \ python3-setuptools \ python3-pip \ - # python3.7 \ - # libpython3.7 \ + python3-psycopg2 \ + python3-ldap \ + python3-libsass \ + python3-lxml \ + python3-pillow \ + python3-pypdf2 \ + python3-psutil \ + python3-asn1crypto \ curl \ - gnupg \ - libpq-dev \ - libsasl2-2 \ - libldap-2.4-2 \ - libxml2 \ - libxmlsec1 \ - libxslt1.1 \ + # libpq-dev \ + # libsasl2-2 \ + # libldap-2.4-2 \ + # libxml2 \ + # libxmlsec1 \ + # libxslt1.1 \ sudo \ node-less \ - # python3-yaml \ - && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 \ - # && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 2 \ - && update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1 \ + && echo 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' > etc/apt/sources.list.d/pgdg.list \ + && export GNUPGHOME="$(mktemp -d)" \ + && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ + && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ + && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ + && gpgconf --kill all \ + && rm -rf "$GNUPGHOME" \ + # Download deb for wkhtmltox + && echo "Downloading https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_${ARCH}.deb" \ + && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_${ARCH}.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + ./wkhtmltox.deb \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1 \ && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 2 \ - && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb \ - && apt-get install -y --no-install-recommends ./wkhtmltox.deb \ && rm -rf /var/lib/apt/lists/* wkhtmltox.deb -# Install latest postgresql-client -RUN set -x; \ - echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ - && export GNUPGHOME="$(mktemp -d)" \ - && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ - && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ - && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ - && gpgconf --kill all \ - && rm -rf "$GNUPGHOME" \ - && apt-get update \ - && apt-get install -y postgresql-client \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH=/usr/local/bin:$PATH +ENV PATH=/var/lib/odoo/.local/bin:/usr/local/bin:$PATH # Install Odoo Including things from sources ENV ODOO_VERSION 14.0 -ENV ODOO_RELEASE=20210212 +ENV ODOO_RELEASE=20221104 ARG ODOO_ARCHIVE=odoo_${ODOO_VERSION}.${ODOO_RELEASE}.tar.gz RUN set -x; \ apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ + # build-essential \ + # python3.8-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ && pip install -U pip \ && /usr/bin/env pip install \ - psycogreen \ + odoo-tools \ \ - && curl -o requirements.txt https://raw.githubusercontent.com/odoo/odoo/${ODOO_VERSION}/requirements.txt \ - && pip install -r requirements.txt \ - && /usr/bin/env pip install https://nightly.odoo.com/${ODOO_VERSION}/nightly/src/${ODOO_ARCHIVE} \ + && odootools manage setup --release "20221104" --repo "https://github.com/odoo/odoo.git" --ref "" "14.0" \ && cd / \ && apt-get --purge remove -y \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ + # build-essential \ + # python3.8-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* - -VOLUME /etc/odoo -VOLUME /var/lib/odoo + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /root/.cache -COPY ./odoo.conf /etc/odoo/ -COPY ./entrypoint.py / -COPY ./sudo-entrypoint.py / -COPY ./find_modules.py /scripts/find_modules.py -COPY ./prepare_project.py /scripts/prepare_project.py +# COPY ./odoo.conf /etc/odoo/ +# COPY ./entrypoint.py / +# COPY ./sudo-entrypoint.py / ARG UID=1000 ARG GID=1000 @@ -95,12 +99,15 @@ ARG GID=1000 RUN mkdir /addons \ && groupadd -r -g ${GID} odoo \ && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ - && chown odoo /etc/odoo/odoo.conf \ + # && chown odoo /etc/odoo/odoo.conf \ && chown -R odoo:odoo /addons \ - && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /sudo-entrypoint.py" >> /etc/sudoers + && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /usr/local/bin/odootools" >> /etc/sudoers + +# VOLUME /etc/odoo +VOLUME /var/lib/odoo -ENV ODOO_RC /etc/odoo/odoo.conf -ENV ODOO_BASE_PATH /usr/local/lib/python3.6/dist-packages/odoo/addons +ENV ODOO_RC /var/lib/odoo/odoo.conf +ENV ODOO_BASE_PATH /usr/local/lib/python3.8/dist-packages/odoo/addons # Env variable defined to monitor the kind of service running # it could be a staging/production/test or anything and undefined # is the default in case we need to know servers that aren't correctly @@ -113,19 +120,19 @@ expose 8071 USER odoo LABEL version="14.0" -LABEL release="20210212" +LABEL release="20221104" -LABEL org.opencontainers.image.created="2021-06-25T17:55:41.197637" +LABEL org.opencontainers.image.created="2022-11-04T20:06:13.804232" LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" -LABEL org.opencontainers.image.authors="Archeti " +LABEL org.opencontainers.image.authors="OdooPlus " LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.version="14.0" -LABEL org.opencontainers.image.vendor="ArcheTI" +LABEL org.opencontainers.image.vendor="OdooPlus" LABEL org.opencontainers.image.ref.name="14.0" LABEL org.opencontainers.image.title="Odoo 14.0" LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." -ENTRYPOINT ["/entrypoint.py"] +ENTRYPOINT ["odootools", "--log-level", "INFO", "entrypoint", "user"] cmd ["odoo"] diff --git a/14.0/entrypoint.py b/14.0/entrypoint.py index f2ce4654b..bb0ffddc3 100755 --- a/14.0/entrypoint.py +++ b/14.0/entrypoint.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # import argparse +from __future__ import print_function import time import shlex import subprocess @@ -17,25 +18,9 @@ from passlib.context import CryptContext from contextlib import contextmanager -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - -try: - from pip._internal.network.session import PipSession - from pip._internal.req.req_file import parse_requirements - from pip._internal.req.constructors import ( - install_req_from_parsed_requirement - ) -except Exception: - from pip.download import PipSession - from pip.req.req_file import parse_requirements - - def install_req_from_parsed_requirement(req): - return req - -from collections import defaultdict +from odoo_tools.compat import Path +from odoo_tools.modules.search import find_addons_paths, find_modules_paths +from odoo_tools.configuration.pips import merge_requirements try: # python3 @@ -68,63 +53,6 @@ def quote(s): return "'" + s.replace("'", "'\"'\"'") + "'" -class Requirement(object): - def __init__(self): - self.extras = set() - self.specifiers = set() - self.links = set() - self.editable = False - - -def merge_requirements(files): - requirements = defaultdict(lambda: Requirement()) - links = set() - - for filename in files: - f_requirements = parse_requirements( - str(filename), session=PipSession() - ) - for parsed_requirement in f_requirements: - requirement = install_req_from_parsed_requirement( - parsed_requirement - ) - - if requirement.markers and not requirement.markers.evaluate(): - continue - - if not hasattr(requirement.req, 'name'): - links.add(requirement.link.url) - break - name = requirement.req.name - specifiers = requirement.req.specifier - extras = requirement.req.extras - requirements[name].extras |= set(extras) - requirements[name].specifiers |= set(specifiers) - if requirement.link: - requirements[name].links |= {requirement.link.url} - requirements[name].editable |= requirement.editable - - result = [] - for key, value in requirements.items(): - if value.links: - result.append("%s" % value.links.pop()) - elif not value.extras: - result.append( - "%s %s" % (key, ",".join(map(str, value.specifiers))) - ) - else: - result.append("%s [%s] %s" % ( - key, - ",".join(map(str, value.extras)), - ",".join(map(str, value.specifiers)) - )) - - for link in links: - result.append(link) - - return "\n".join(result) - - def pipe(args): """ Call the process with std(in,out,err) @@ -178,42 +106,12 @@ def call_sudo_entrypoint(): return ret -def get_all_manifests(): - odoo_manifests = glob.glob('/addons/**/__manifest__.py', recursive=True) - erp_manifest = glob.glob('/addons/**/__openerp__.py', recursive=True) - - return odoo_manifests + erp_manifest - - -def module_name(manifest_file): - return path.basename(path.dirname(manifest_file)) - - -def get_module(manifest_file): - with open(manifest_file) as manifest_in: - try: - code = compile(manifest_in.read(), '__manifest__', 'eval') - module = eval(code, {}, {}) - except Exception: - module = {} - - return module - - -def is_installable(manifest): - return manifest.get('installable', True) - - -def is_server_wide(manifest): - return manifest.get('server_wide', False) - - def get_server_wide_modules(manifests): base_server_wide_modules = ['base', 'web'] custom_server_wide_modules = [ - module_name(filename) - for filename, module in manifests.items() - if is_server_wide(module) + manifest.technical_name + for manifest in manifests + if manifest.server_wide ] return base_server_wide_modules + custom_server_wide_modules @@ -246,7 +144,12 @@ def install_python_dependencies(valid_paths): req_file = Path('/var/lib/odoo/requirements.txt') with req_file.open('w') as fout: data = merge_requirements(requirement_files) - fout.write(data) + data = "\n".join(data) + + try: + fout.write(data) + except Exception: + fout.write(data.decode('utf-8')) for requirements in requirement_files: print("Installing python packages from %s" % requirements) @@ -338,65 +241,44 @@ def get_extra_paths(): ] -def get_valid_paths(): - base_addons = os.environ.get('ODOO_BASE_PATH') +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] - # addons = os.listdir('/addons') - valid_paths = [base_addons] +def get_valid_paths(): + base_addons = os.environ['ODOO_BASE_PATH'] - addons_paths = get_dirs('/addons') - addons_paths += get_extra_paths() + base_addons_paths = [base_addons, '/addons'] + base_addons_paths += get_extra_paths() - for addons_path in addons_paths: - print("Lookup addons in %s" % addons_path) - flush_streams() - addons = get_dirs(addons_path) - for addon in addons: - files = os.listdir(addon) - if ( - '__init__.py' in files and - '__manifest__.py' in files or - '__openerp__.py' in files - ): - valid_paths.append(addons_path) - print("Addons found !") - flush_streams() - break - if addons_path in valid_paths: - continue - else: - print("No addons found in path. Skipping...") - flush_streams() + valid_paths = find_addons_paths(base_addons_paths) + + for valid_path in valid_paths: + print("Found installable modules in {}".format(valid_path)) return valid_paths def setup_addons_paths(config, valid_paths): - config.set('options', 'addons_path', ",".join(valid_paths)) + config.set('options', 'addons_path', ",".join([ + str(path) for path in valid_paths + ])) flush_streams() -def setup_server_wide_modules(config): +def setup_server_wide_modules(config, valid_paths): print("Searching for server wide modules") - all_manifests = get_all_manifests() + filters = set(['installable']) + all_manifests = find_modules_paths(valid_paths, filters) - print("Found %s modules" % (len(all_manifests))) + print("Found %s installable modules " % (len(all_manifests),)) - manifests = { - filename: get_module(filename) - for filename in all_manifests - } - - installable_manifests = { - filename: module - for filename, module in manifests.items() - if is_installable(module) - } - - print("Found %s installable modules " % (len(installable_manifests),)) - - server_wide_modules = get_server_wide_modules(installable_manifests) + server_wide_modules = get_server_wide_modules(all_manifests) if len(server_wide_modules) > 2: modules = ",".join(server_wide_modules) @@ -408,6 +290,36 @@ def setup_server_wide_modules(config): flush_streams() +def convert_value(name, value): + if value in ['True', 'False']: + return value == 'True' + + return value + + +def setup_env_config(config): + try: + from odoo.tools import config as odoo_config + except ImportError: + from openerp.tools import config as odoo_config + + params_by_name = {} + for key, opt in odoo_config.casts.items(): + value_opt = opt.get_opt_string() + value_opt = value_opt.upper().replace('--', 'ODOO_') + value_opt = value_opt.replace('-', '_') + params_by_name[value_opt] = key + + for key, value in os.environ.items(): + if not key.startswith('ODOO_'): + continue + + if key in params_by_name: + config_name = params_by_name[key] + converted_value = convert_value(key, value) + config.set('options', config_name, converted_value) + + def setup_environ(config): print("Configuring environment variables for postgresql") @@ -499,12 +411,39 @@ def wait_postgresql(): @contextmanager def get_config(config_path): config = ConfigParser() - config.read(config_path) + config.read(str(config_path)) yield config - with config_path.open('w') as out: - config.write(out) + try: + with config_path.open('w') as out: + config.write(out) + except Exception: + with config_path.open('wb') as out: + config.write(out) + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for vpath in paths: + cur_path = Path(vpath) + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(vpath) + + return res_paths def main(): @@ -518,7 +457,9 @@ def main(): sys.exit(ret) # Install python packages with pip in user home - valid_paths = get_valid_paths() + orig_valid_paths = get_valid_paths() + excluded_paths = get_excluded_paths() + valid_paths = filter_valid_paths(orig_valid_paths, excluded_paths) config_path = Path(os.environ.get('ODOO_RC', '/etc/odoo/odoo.cfg')) @@ -528,7 +469,8 @@ def main(): install_master_password(config) setup_environ(config) setup_addons_paths(config, valid_paths) - setup_server_wide_modules(config) + setup_server_wide_modules(config, valid_paths) + setup_env_config(config) if not os.environ.get('ODOO_SKIP_POSTGRES_WAIT'): wait_postgresql() diff --git a/14.0/find_modules.py b/14.0/find_modules.py deleted file mode 100644 index 9cb943748..000000000 --- a/14.0/find_modules.py +++ /dev/null @@ -1,76 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -from itertools import chain - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - "-p", - dest='paths', - action="append", - help="Location in which to search", - default=[] - ) - - parser.add_option( - "--only-name", - dest="only_name", - action="store_true", - help="Only display module name instead of path", - default=False, - ) - - parser.add_option( - '--csv', - dest="is_csv", - action="store_true", - help="Output as a comma separated list", - default=False - ) - - return parser - - -def find_modules(options, path): - modules = set() - - path = Path.cwd() / path - - erp_manifest = '__openerp__.py' - odoo_manifest = '__manifest__.py' - - manifest_globs = chain( - path.glob('**/{}'.format(erp_manifest)), - path.glob('**/{}'.format(odoo_manifest)), - ) - - for path in manifest_globs: - rel_path = path.parent.relative_to(Path.cwd()) - if options.only_name: - modules.add(rel_path.name) - else: - modules.add(str(rel_path)) - - return modules - - -def main(options, args): - modules = set() - for path in options.paths: - modules = modules.union(find_modules(options, path)) - - return modules - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - modules = main(options, args) - - if not options.is_csv: - for module in modules: - print(module) - else: - print(",".join(modules), end="") diff --git a/14.0/platform.sh b/14.0/platform.sh new file mode 100644 index 000000000..fdf5e42eb --- /dev/null +++ b/14.0/platform.sh @@ -0,0 +1,13 @@ +PLATFORM=`uname -m` + +if [ $PLATFORM = "aarch64" ] +then + echo -n "arm64" +else +if [ $PLATFORM = "x86_64" ] +then + echo -n "amd64" +else + echo -n $PLATFORM +fi +fi diff --git a/14.0/prepare_project.py b/14.0/prepare_project.py deleted file mode 100644 index 74a53f2e0..000000000 --- a/14.0/prepare_project.py +++ /dev/null @@ -1,198 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -import toml -from giturlparse import parse -from contextlib import contextmanager -import os -from subprocess import run -from urllib.parse import urlparse - - -def get_services(services): - return { - service.get('name'): service - for service in services.get('services') - } - - -def addons_by_project(options, addons): - res = {} - - for addon in addons: - if addon['url'] == 'self': - if options.ignore_self: - continue - parsed = parse(options.url) - else: - parsed = parse(addon['url']) - - auth = parsed.protocol in ['git', 'ssh'] - - res[parsed.repo] = dict( - addon, - url=parsed.url2https, - auth=auth - ) - - return res - - -def merge_addons(options, base, other): - base_addons = addons_by_project(options, base) - other_addons = addons_by_project(options, other) - - for name, addon in other_addons.items(): - if name not in base_addons: - base_addons[name] = addon - else: - base_addons[name] = dict(base_addons[name], **addon) - - return [ - addon - for addon in base_addons.values() - ] - - -def merge_services(options, base, other): - basic_inherit = dict(base, **other) - - if base.get('addons') or other.get('addons'): - basic_inherit['addons'] = merge_addons( - options, - base.get('addons', []), - other.get('addons', []) - ) - - return basic_inherit - - -def compile_service(options, services, name): - service = services.get(name, {}) - if 'inherit' in service: - merge_service = compile_service(options, services, service['inherit']) - service = merge_services(options, merge_service, service) - - return service - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - '-f', - '--file', - dest="file", - help="Input File" - ) - - parser.add_option( - '--url', - dest='url', - help="Url of self project" - ) - - parser.add_option( - '-o', - '--output', - dest="output_directory", - help="Output Directory" - ) - - parser.add_option( - '-e', - dest="env", - help="Environment to prepare" - ) - - parser.add_option( - '--username', - dest="username", - help="Username to replace with", - ) - - parser.add_option( - '--password', - dest="password", - help="password to set on https urls" - ) - - parser.add_option( - '-b', - '--branch', - dest="branch", - help="Default branch if no ref is defined" - ) - - parser.add_option( - '--ignore-self', - dest="ignore_self", - action="store_true", - help="Ignore self url as it's already fetched", - default=False - ) - - return parser - - -@contextmanager -def cd(directory): - cwd = Path.cwd() - try: - os.chdir(directory) - yield - finally: - os.chdir(cwd) - - -def fetch_addons(options, addon): - parsed = parse(addon['url']) - - url = urlparse(parsed.url2https) - - if addon['auth'] and options.username and options.password: - url = url._replace( - netloc="{}:{}@{}".format( - options.username, - options.password, - url.netloc - ) - ) - - repo_path = Path.cwd() / options.output_directory / parsed.repo - - repo_path.mkdir(exist_ok=True) - - with cd(repo_path): - run(['git', 'init']) - run(['git', 'remote', 'add', 'origin', url.geturl()]) - - ref = addon.get('commit') or addon.get('branch') or options.branch - - if ref: - run(['git', 'fetch', 'origin', ref]) - else: - run(['git', 'fetch', 'origin']) - - run(['git', 'checkout', 'FETCH_HEAD']) - run(['git', 'remote', 'remove', 'origin']) - - -def main(options, args): - with Path(options.file).open('r') as fin: - services = toml.loads(fin.read()) - - by_name = get_services(services) - - outputdir = Path(options.output_directory) - outputdir.mkdir(exist_ok=True) - - service = compile_service(options, by_name, options.env) - - for addons in service.get('addons', []): - fetch_addons(options, addons) - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - main(options, args) diff --git a/14.0/sudo-entrypoint.py b/14.0/sudo-entrypoint.py index 98779baae..916239170 100755 --- a/14.0/sudo-entrypoint.py +++ b/14.0/sudo-entrypoint.py @@ -12,6 +12,7 @@ from os import path from os.path import expanduser import shutil +import six try: from pathlib import Path @@ -69,6 +70,37 @@ def get_addons_paths(): ] +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for cur_path in paths: + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(cur_path) + + return res_paths + + def install_apt_packages(): """ Install debian dependencies. @@ -76,26 +108,34 @@ def install_apt_packages(): package_list = set() paths = get_addons_paths() + excluded_paths = get_excluded_paths() + paths = filter_valid_paths(paths, excluded_paths) print("Looking up for packages in {}".format(paths)) for addons_path in paths: for packages in addons_path.glob('**/apt-packages.txt'): print("Installing packages from %s" % packages) - with open(packages, 'r') as pack_file: - lines = [line.strip() for line in pack_file] + with packages.open('r') as pack_file: + lines = [ + six.ensure_text(line).strip() + for line in pack_file + if six.ensure_text(line).strip() + ] package_list.update(set(lines)) extras = os.environ.get('EXTRA_APT_PACKAGES', '') - print(f"Adding extra packages {extras}") + print("Adding extra packages {extras}".format(extras=extras)) if extras: for package in extras.split(','): - if not package: + if not package.strip(): continue - package_list.add(package) + package_list.add( + six.ensure_text(package).strip() + ) if len(package_list) > 0: - print(f"Installing {package_list}") + print("Installing {package_list}".format(package_list=package_list)) ret = pipe(['apt-get', 'update']) # Something went wrong, stop the service as it's failing diff --git a/14.3/Dockerfile b/14.3/Dockerfile deleted file mode 100644 index 50fd334af..000000000 --- a/14.3/Dockerfile +++ /dev/null @@ -1,131 +0,0 @@ -FROM ubuntu:bionic -LABEL maintainer="Archeti " - -# Generate locale C.UTF-8 for postgres and general locale data -ENV LANG C.UTF-8 -ARG DEBIAN_FRONTEND=noninteractive - -# Install some dependencies python3.7 -RUN set -x; \ - apt-get update \ - && apt-get install -y --no-install-recommends \ - python3-wheel \ - python3-setuptools \ - python3-pip \ - # python3.7 \ - # libpython3.7 \ - curl \ - gnupg \ - libpq-dev \ - libsasl2-2 \ - libldap-2.4-2 \ - libxml2 \ - libxmlsec1 \ - libxslt1.1 \ - sudo \ - node-less \ - # python3-yaml \ - && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 \ - # && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 2 \ - && update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1 \ - && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 2 \ - && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb \ - && apt-get install -y --no-install-recommends ./wkhtmltox.deb \ - && rm -rf /var/lib/apt/lists/* wkhtmltox.deb - -# Install latest postgresql-client -RUN set -x; \ - echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ - && export GNUPGHOME="$(mktemp -d)" \ - && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ - && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ - && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ - && gpgconf --kill all \ - && rm -rf "$GNUPGHOME" \ - && apt-get update \ - && apt-get install -y postgresql-client \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH=/usr/local/bin:$PATH -# Install Odoo Including things from sources -ENV ODOO_VERSION 14.3 -ENV ODOO_RELEASE=20210402 -ARG ODOO_ARCHIVE=odoo_14.3alpha1.${ODOO_RELEASE}.tar.gz -RUN set -x; \ - apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ - && pip install -U pip \ - && /usr/bin/env pip install \ - psycogreen \ - \ - && curl -o requirements.txt https://raw.githubusercontent.com/odoo/odoo/master/requirements.txt \ - && pip install -r requirements.txt \ - && /usr/bin/env pip install https://nightly.odoo.com/master/nightly/src/${ODOO_ARCHIVE} \ - && cd / \ - && apt-get --purge remove -y \ - build-essential \ - python3.6-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* - -VOLUME /etc/odoo -VOLUME /var/lib/odoo - -COPY ./odoo.conf /etc/odoo/ -COPY ./entrypoint.py / -COPY ./sudo-entrypoint.py / -COPY ./find_modules.py /scripts/find_modules.py -COPY ./prepare_project.py /scripts/prepare_project.py - -ARG UID=1000 -ARG GID=1000 - -RUN mkdir /addons \ - && groupadd -r -g ${GID} odoo \ - && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ - && chown odoo /etc/odoo/odoo.conf \ - && chown -R odoo:odoo /addons \ - && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /sudo-entrypoint.py" >> /etc/sudoers - -ENV ODOO_RC /etc/odoo/odoo.conf -ENV ODOO_BASE_PATH /usr/local/lib/python3.6/dist-packages/odoo/addons -# Env variable defined to monitor the kind of service running -# it could be a staging/production/test or anything and undefined -# is the default in case we need to know servers that aren't correctly -# defined -ENV DEPLOYMENT_AREA undefined - -expose 8069 -expose 8071 - -USER odoo - -LABEL version="14.3" -LABEL release="20210402" - -LABEL org.opencontainers.image.created="2021-06-25T17:55:41.204087" -LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" -LABEL org.opencontainers.image.authors="Archeti " -LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" -LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" -LABEL org.opencontainers.image.version="14.3" -LABEL org.opencontainers.image.vendor="ArcheTI" -LABEL org.opencontainers.image.ref.name="14.3" -LABEL org.opencontainers.image.title="Odoo 14.3" -LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." - -ENTRYPOINT ["/entrypoint.py"] - -cmd ["odoo"] diff --git a/14.3/find_modules.py b/14.3/find_modules.py deleted file mode 100644 index 9cb943748..000000000 --- a/14.3/find_modules.py +++ /dev/null @@ -1,76 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -from itertools import chain - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - "-p", - dest='paths', - action="append", - help="Location in which to search", - default=[] - ) - - parser.add_option( - "--only-name", - dest="only_name", - action="store_true", - help="Only display module name instead of path", - default=False, - ) - - parser.add_option( - '--csv', - dest="is_csv", - action="store_true", - help="Output as a comma separated list", - default=False - ) - - return parser - - -def find_modules(options, path): - modules = set() - - path = Path.cwd() / path - - erp_manifest = '__openerp__.py' - odoo_manifest = '__manifest__.py' - - manifest_globs = chain( - path.glob('**/{}'.format(erp_manifest)), - path.glob('**/{}'.format(odoo_manifest)), - ) - - for path in manifest_globs: - rel_path = path.parent.relative_to(Path.cwd()) - if options.only_name: - modules.add(rel_path.name) - else: - modules.add(str(rel_path)) - - return modules - - -def main(options, args): - modules = set() - for path in options.paths: - modules = modules.union(find_modules(options, path)) - - return modules - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - modules = main(options, args) - - if not options.is_csv: - for module in modules: - print(module) - else: - print(",".join(modules), end="") diff --git a/14.3/prepare_project.py b/14.3/prepare_project.py deleted file mode 100644 index 74a53f2e0..000000000 --- a/14.3/prepare_project.py +++ /dev/null @@ -1,198 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -import toml -from giturlparse import parse -from contextlib import contextmanager -import os -from subprocess import run -from urllib.parse import urlparse - - -def get_services(services): - return { - service.get('name'): service - for service in services.get('services') - } - - -def addons_by_project(options, addons): - res = {} - - for addon in addons: - if addon['url'] == 'self': - if options.ignore_self: - continue - parsed = parse(options.url) - else: - parsed = parse(addon['url']) - - auth = parsed.protocol in ['git', 'ssh'] - - res[parsed.repo] = dict( - addon, - url=parsed.url2https, - auth=auth - ) - - return res - - -def merge_addons(options, base, other): - base_addons = addons_by_project(options, base) - other_addons = addons_by_project(options, other) - - for name, addon in other_addons.items(): - if name not in base_addons: - base_addons[name] = addon - else: - base_addons[name] = dict(base_addons[name], **addon) - - return [ - addon - for addon in base_addons.values() - ] - - -def merge_services(options, base, other): - basic_inherit = dict(base, **other) - - if base.get('addons') or other.get('addons'): - basic_inherit['addons'] = merge_addons( - options, - base.get('addons', []), - other.get('addons', []) - ) - - return basic_inherit - - -def compile_service(options, services, name): - service = services.get(name, {}) - if 'inherit' in service: - merge_service = compile_service(options, services, service['inherit']) - service = merge_services(options, merge_service, service) - - return service - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - '-f', - '--file', - dest="file", - help="Input File" - ) - - parser.add_option( - '--url', - dest='url', - help="Url of self project" - ) - - parser.add_option( - '-o', - '--output', - dest="output_directory", - help="Output Directory" - ) - - parser.add_option( - '-e', - dest="env", - help="Environment to prepare" - ) - - parser.add_option( - '--username', - dest="username", - help="Username to replace with", - ) - - parser.add_option( - '--password', - dest="password", - help="password to set on https urls" - ) - - parser.add_option( - '-b', - '--branch', - dest="branch", - help="Default branch if no ref is defined" - ) - - parser.add_option( - '--ignore-self', - dest="ignore_self", - action="store_true", - help="Ignore self url as it's already fetched", - default=False - ) - - return parser - - -@contextmanager -def cd(directory): - cwd = Path.cwd() - try: - os.chdir(directory) - yield - finally: - os.chdir(cwd) - - -def fetch_addons(options, addon): - parsed = parse(addon['url']) - - url = urlparse(parsed.url2https) - - if addon['auth'] and options.username and options.password: - url = url._replace( - netloc="{}:{}@{}".format( - options.username, - options.password, - url.netloc - ) - ) - - repo_path = Path.cwd() / options.output_directory / parsed.repo - - repo_path.mkdir(exist_ok=True) - - with cd(repo_path): - run(['git', 'init']) - run(['git', 'remote', 'add', 'origin', url.geturl()]) - - ref = addon.get('commit') or addon.get('branch') or options.branch - - if ref: - run(['git', 'fetch', 'origin', ref]) - else: - run(['git', 'fetch', 'origin']) - - run(['git', 'checkout', 'FETCH_HEAD']) - run(['git', 'remote', 'remove', 'origin']) - - -def main(options, args): - with Path(options.file).open('r') as fin: - services = toml.loads(fin.read()) - - by_name = get_services(services) - - outputdir = Path(options.output_directory) - outputdir.mkdir(exist_ok=True) - - service = compile_service(options, by_name, options.env) - - for addons in service.get('addons', []): - fetch_addons(options, addons) - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - main(options, args) diff --git a/15.0-nightly/Dockerfile b/15.0-nightly/Dockerfile new file mode 100644 index 000000000..0376e3e3b --- /dev/null +++ b/15.0-nightly/Dockerfile @@ -0,0 +1,138 @@ +FROM ubuntu:focal +LABEL maintainer="Archeti " + +# Generate locale C.UTF-8 for postgres and general locale data +ENV LANG C.UTF-8 +ARG DEBIAN_FRONTEND=noninteractive + +COPY ./platform.sh /platform.sh +# Install some dependencies python3.7 +RUN set -x; \ + export ARCH=`sh /platform.sh` \ + # Setup deb source for postgresql client + && apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg \ + python3.8 \ + python3-wheel \ + python3-setuptools \ + python3-pip \ + python3-psycopg2 \ + python3-ldap \ + python3-libsass \ + python3-lxml \ + python3-pillow \ + python3-pypdf2 \ + python3-psutil \ + python3-asn1crypto \ + curl \ + # libpq-dev \ + # libsasl2-2 \ + # libldap-2.4-2 \ + # libxml2 \ + # libxmlsec1 \ + # libxslt1.1 \ + sudo \ + node-less \ + && echo 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' > etc/apt/sources.list.d/pgdg.list \ + && export GNUPGHOME="$(mktemp -d)" \ + && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ + && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ + && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ + && gpgconf --kill all \ + && rm -rf "$GNUPGHOME" \ + # Download deb for wkhtmltox + && echo "Downloading https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_${ARCH}.deb" \ + && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_${ARCH}.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + ./wkhtmltox.deb \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1 \ + && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 2 \ + && rm -rf /var/lib/apt/lists/* wkhtmltox.deb + +ENV PATH=/var/lib/odoo/.local/bin:/usr/local/bin:$PATH +# Install Odoo Including things from sources +ENV ODOO_VERSION 15.0 +ENV ODOO_RELEASE= +ARG ODOO_ARCHIVE=odoo_${ODOO_VERSION}.${ODOO_RELEASE}.tar.gz +RUN set -x; \ + apt-get update \ + && apt-get install -y --no-install-recommends \ + # build-essential \ + # python3.8-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ + && pip install -U pip \ + && /usr/bin/env pip install \ + odoo-tools \ + \ + && odootools manage setup --release "" --repo "https://github.com/odoo/odoo.git" --ref "" "15.0" \ + && cd / \ + && apt-get --purge remove -y \ + # build-essential \ + # python3.8-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /root/.cache + +# COPY ./odoo.conf /etc/odoo/ +# COPY ./entrypoint.py / +# COPY ./sudo-entrypoint.py / + +ARG UID=1000 +ARG GID=1000 + +RUN mkdir /addons \ + && groupadd -r -g ${GID} odoo \ + && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ + # && chown odoo /etc/odoo/odoo.conf \ + && chown -R odoo:odoo /addons \ + && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /usr/local/bin/odootools" >> /etc/sudoers + +# VOLUME /etc/odoo +VOLUME /var/lib/odoo + +ENV ODOO_RC /var/lib/odoo/odoo.conf +ENV ODOO_BASE_PATH /usr/local/lib/python3.8/dist-packages/odoo/addons +# Env variable defined to monitor the kind of service running +# it could be a staging/production/test or anything and undefined +# is the default in case we need to know servers that aren't correctly +# defined +ENV DEPLOYMENT_AREA undefined + +expose 8069 +expose 8071 + +USER odoo + +LABEL version="15.0" +LABEL release="" + +LABEL org.opencontainers.image.created="2022-11-04T20:06:13.825965" +LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" +LABEL org.opencontainers.image.authors="OdooPlus " +LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" +LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" +LABEL org.opencontainers.image.version="15.0" +LABEL org.opencontainers.image.vendor="OdooPlus" +LABEL org.opencontainers.image.ref.name="15.0-nightly" +LABEL org.opencontainers.image.title="Odoo 15.0" +LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." + +ENTRYPOINT ["odootools", "--log-level", "INFO", "entrypoint", "user"] + +cmd ["odoo"] diff --git a/14.0-tiny/entrypoint.py b/15.0-nightly/entrypoint.py similarity index 69% rename from 14.0-tiny/entrypoint.py rename to 15.0-nightly/entrypoint.py index f2ce4654b..bb0ffddc3 100755 --- a/14.0-tiny/entrypoint.py +++ b/15.0-nightly/entrypoint.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # import argparse +from __future__ import print_function import time import shlex import subprocess @@ -17,25 +18,9 @@ from passlib.context import CryptContext from contextlib import contextmanager -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - -try: - from pip._internal.network.session import PipSession - from pip._internal.req.req_file import parse_requirements - from pip._internal.req.constructors import ( - install_req_from_parsed_requirement - ) -except Exception: - from pip.download import PipSession - from pip.req.req_file import parse_requirements - - def install_req_from_parsed_requirement(req): - return req - -from collections import defaultdict +from odoo_tools.compat import Path +from odoo_tools.modules.search import find_addons_paths, find_modules_paths +from odoo_tools.configuration.pips import merge_requirements try: # python3 @@ -68,63 +53,6 @@ def quote(s): return "'" + s.replace("'", "'\"'\"'") + "'" -class Requirement(object): - def __init__(self): - self.extras = set() - self.specifiers = set() - self.links = set() - self.editable = False - - -def merge_requirements(files): - requirements = defaultdict(lambda: Requirement()) - links = set() - - for filename in files: - f_requirements = parse_requirements( - str(filename), session=PipSession() - ) - for parsed_requirement in f_requirements: - requirement = install_req_from_parsed_requirement( - parsed_requirement - ) - - if requirement.markers and not requirement.markers.evaluate(): - continue - - if not hasattr(requirement.req, 'name'): - links.add(requirement.link.url) - break - name = requirement.req.name - specifiers = requirement.req.specifier - extras = requirement.req.extras - requirements[name].extras |= set(extras) - requirements[name].specifiers |= set(specifiers) - if requirement.link: - requirements[name].links |= {requirement.link.url} - requirements[name].editable |= requirement.editable - - result = [] - for key, value in requirements.items(): - if value.links: - result.append("%s" % value.links.pop()) - elif not value.extras: - result.append( - "%s %s" % (key, ",".join(map(str, value.specifiers))) - ) - else: - result.append("%s [%s] %s" % ( - key, - ",".join(map(str, value.extras)), - ",".join(map(str, value.specifiers)) - )) - - for link in links: - result.append(link) - - return "\n".join(result) - - def pipe(args): """ Call the process with std(in,out,err) @@ -178,42 +106,12 @@ def call_sudo_entrypoint(): return ret -def get_all_manifests(): - odoo_manifests = glob.glob('/addons/**/__manifest__.py', recursive=True) - erp_manifest = glob.glob('/addons/**/__openerp__.py', recursive=True) - - return odoo_manifests + erp_manifest - - -def module_name(manifest_file): - return path.basename(path.dirname(manifest_file)) - - -def get_module(manifest_file): - with open(manifest_file) as manifest_in: - try: - code = compile(manifest_in.read(), '__manifest__', 'eval') - module = eval(code, {}, {}) - except Exception: - module = {} - - return module - - -def is_installable(manifest): - return manifest.get('installable', True) - - -def is_server_wide(manifest): - return manifest.get('server_wide', False) - - def get_server_wide_modules(manifests): base_server_wide_modules = ['base', 'web'] custom_server_wide_modules = [ - module_name(filename) - for filename, module in manifests.items() - if is_server_wide(module) + manifest.technical_name + for manifest in manifests + if manifest.server_wide ] return base_server_wide_modules + custom_server_wide_modules @@ -246,7 +144,12 @@ def install_python_dependencies(valid_paths): req_file = Path('/var/lib/odoo/requirements.txt') with req_file.open('w') as fout: data = merge_requirements(requirement_files) - fout.write(data) + data = "\n".join(data) + + try: + fout.write(data) + except Exception: + fout.write(data.decode('utf-8')) for requirements in requirement_files: print("Installing python packages from %s" % requirements) @@ -338,65 +241,44 @@ def get_extra_paths(): ] -def get_valid_paths(): - base_addons = os.environ.get('ODOO_BASE_PATH') +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] - # addons = os.listdir('/addons') - valid_paths = [base_addons] +def get_valid_paths(): + base_addons = os.environ['ODOO_BASE_PATH'] - addons_paths = get_dirs('/addons') - addons_paths += get_extra_paths() + base_addons_paths = [base_addons, '/addons'] + base_addons_paths += get_extra_paths() - for addons_path in addons_paths: - print("Lookup addons in %s" % addons_path) - flush_streams() - addons = get_dirs(addons_path) - for addon in addons: - files = os.listdir(addon) - if ( - '__init__.py' in files and - '__manifest__.py' in files or - '__openerp__.py' in files - ): - valid_paths.append(addons_path) - print("Addons found !") - flush_streams() - break - if addons_path in valid_paths: - continue - else: - print("No addons found in path. Skipping...") - flush_streams() + valid_paths = find_addons_paths(base_addons_paths) + + for valid_path in valid_paths: + print("Found installable modules in {}".format(valid_path)) return valid_paths def setup_addons_paths(config, valid_paths): - config.set('options', 'addons_path', ",".join(valid_paths)) + config.set('options', 'addons_path', ",".join([ + str(path) for path in valid_paths + ])) flush_streams() -def setup_server_wide_modules(config): +def setup_server_wide_modules(config, valid_paths): print("Searching for server wide modules") - all_manifests = get_all_manifests() + filters = set(['installable']) + all_manifests = find_modules_paths(valid_paths, filters) - print("Found %s modules" % (len(all_manifests))) + print("Found %s installable modules " % (len(all_manifests),)) - manifests = { - filename: get_module(filename) - for filename in all_manifests - } - - installable_manifests = { - filename: module - for filename, module in manifests.items() - if is_installable(module) - } - - print("Found %s installable modules " % (len(installable_manifests),)) - - server_wide_modules = get_server_wide_modules(installable_manifests) + server_wide_modules = get_server_wide_modules(all_manifests) if len(server_wide_modules) > 2: modules = ",".join(server_wide_modules) @@ -408,6 +290,36 @@ def setup_server_wide_modules(config): flush_streams() +def convert_value(name, value): + if value in ['True', 'False']: + return value == 'True' + + return value + + +def setup_env_config(config): + try: + from odoo.tools import config as odoo_config + except ImportError: + from openerp.tools import config as odoo_config + + params_by_name = {} + for key, opt in odoo_config.casts.items(): + value_opt = opt.get_opt_string() + value_opt = value_opt.upper().replace('--', 'ODOO_') + value_opt = value_opt.replace('-', '_') + params_by_name[value_opt] = key + + for key, value in os.environ.items(): + if not key.startswith('ODOO_'): + continue + + if key in params_by_name: + config_name = params_by_name[key] + converted_value = convert_value(key, value) + config.set('options', config_name, converted_value) + + def setup_environ(config): print("Configuring environment variables for postgresql") @@ -499,12 +411,39 @@ def wait_postgresql(): @contextmanager def get_config(config_path): config = ConfigParser() - config.read(config_path) + config.read(str(config_path)) yield config - with config_path.open('w') as out: - config.write(out) + try: + with config_path.open('w') as out: + config.write(out) + except Exception: + with config_path.open('wb') as out: + config.write(out) + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for vpath in paths: + cur_path = Path(vpath) + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(vpath) + + return res_paths def main(): @@ -518,7 +457,9 @@ def main(): sys.exit(ret) # Install python packages with pip in user home - valid_paths = get_valid_paths() + orig_valid_paths = get_valid_paths() + excluded_paths = get_excluded_paths() + valid_paths = filter_valid_paths(orig_valid_paths, excluded_paths) config_path = Path(os.environ.get('ODOO_RC', '/etc/odoo/odoo.cfg')) @@ -528,7 +469,8 @@ def main(): install_master_password(config) setup_environ(config) setup_addons_paths(config, valid_paths) - setup_server_wide_modules(config) + setup_server_wide_modules(config, valid_paths) + setup_env_config(config) if not os.environ.get('ODOO_SKIP_POSTGRES_WAIT'): wait_postgresql() diff --git a/14.0-tiny/odoo.conf b/15.0-nightly/odoo.conf similarity index 100% rename from 14.0-tiny/odoo.conf rename to 15.0-nightly/odoo.conf diff --git a/15.0-nightly/platform.sh b/15.0-nightly/platform.sh new file mode 100644 index 000000000..fdf5e42eb --- /dev/null +++ b/15.0-nightly/platform.sh @@ -0,0 +1,13 @@ +PLATFORM=`uname -m` + +if [ $PLATFORM = "aarch64" ] +then + echo -n "arm64" +else +if [ $PLATFORM = "x86_64" ] +then + echo -n "amd64" +else + echo -n $PLATFORM +fi +fi diff --git a/14.0-tiny/sudo-entrypoint.py b/15.0-nightly/sudo-entrypoint.py similarity index 77% rename from 14.0-tiny/sudo-entrypoint.py rename to 15.0-nightly/sudo-entrypoint.py index 98779baae..916239170 100755 --- a/14.0-tiny/sudo-entrypoint.py +++ b/15.0-nightly/sudo-entrypoint.py @@ -12,6 +12,7 @@ from os import path from os.path import expanduser import shutil +import six try: from pathlib import Path @@ -69,6 +70,37 @@ def get_addons_paths(): ] +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for cur_path in paths: + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(cur_path) + + return res_paths + + def install_apt_packages(): """ Install debian dependencies. @@ -76,26 +108,34 @@ def install_apt_packages(): package_list = set() paths = get_addons_paths() + excluded_paths = get_excluded_paths() + paths = filter_valid_paths(paths, excluded_paths) print("Looking up for packages in {}".format(paths)) for addons_path in paths: for packages in addons_path.glob('**/apt-packages.txt'): print("Installing packages from %s" % packages) - with open(packages, 'r') as pack_file: - lines = [line.strip() for line in pack_file] + with packages.open('r') as pack_file: + lines = [ + six.ensure_text(line).strip() + for line in pack_file + if six.ensure_text(line).strip() + ] package_list.update(set(lines)) extras = os.environ.get('EXTRA_APT_PACKAGES', '') - print(f"Adding extra packages {extras}") + print("Adding extra packages {extras}".format(extras=extras)) if extras: for package in extras.split(','): - if not package: + if not package.strip(): continue - package_list.add(package) + package_list.add( + six.ensure_text(package).strip() + ) if len(package_list) > 0: - print(f"Installing {package_list}") + print("Installing {package_list}".format(package_list=package_list)) ret = pipe(['apt-get', 'update']) # Something went wrong, stop the service as it's failing diff --git a/15.0/Dockerfile b/15.0/Dockerfile new file mode 100644 index 000000000..8e0e6dba6 --- /dev/null +++ b/15.0/Dockerfile @@ -0,0 +1,138 @@ +FROM ubuntu:focal +LABEL maintainer="Archeti " + +# Generate locale C.UTF-8 for postgres and general locale data +ENV LANG C.UTF-8 +ARG DEBIAN_FRONTEND=noninteractive + +COPY ./platform.sh /platform.sh +# Install some dependencies python3.7 +RUN set -x; \ + export ARCH=`sh /platform.sh` \ + # Setup deb source for postgresql client + && apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg \ + python3.8 \ + python3-wheel \ + python3-setuptools \ + python3-pip \ + python3-psycopg2 \ + python3-ldap \ + python3-libsass \ + python3-lxml \ + python3-pillow \ + python3-pypdf2 \ + python3-psutil \ + python3-asn1crypto \ + curl \ + # libpq-dev \ + # libsasl2-2 \ + # libldap-2.4-2 \ + # libxml2 \ + # libxmlsec1 \ + # libxslt1.1 \ + sudo \ + node-less \ + && echo 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' > etc/apt/sources.list.d/pgdg.list \ + && export GNUPGHOME="$(mktemp -d)" \ + && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ + && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ + && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ + && gpgconf --kill all \ + && rm -rf "$GNUPGHOME" \ + # Download deb for wkhtmltox + && echo "Downloading https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_${ARCH}.deb" \ + && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_${ARCH}.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + ./wkhtmltox.deb \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1 \ + && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 2 \ + && rm -rf /var/lib/apt/lists/* wkhtmltox.deb + +ENV PATH=/var/lib/odoo/.local/bin:/usr/local/bin:$PATH +# Install Odoo Including things from sources +ENV ODOO_VERSION 15.0 +ENV ODOO_RELEASE=20221104 +ARG ODOO_ARCHIVE=odoo_${ODOO_VERSION}.${ODOO_RELEASE}.tar.gz +RUN set -x; \ + apt-get update \ + && apt-get install -y --no-install-recommends \ + # build-essential \ + # python3.8-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ + && pip install -U pip \ + && /usr/bin/env pip install \ + odoo-tools \ + \ + && odootools manage setup --release "20221104" --repo "https://github.com/odoo/odoo.git" --ref "" "15.0" \ + && cd / \ + && apt-get --purge remove -y \ + # build-essential \ + # python3.8-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /root/.cache + +# COPY ./odoo.conf /etc/odoo/ +# COPY ./entrypoint.py / +# COPY ./sudo-entrypoint.py / + +ARG UID=1000 +ARG GID=1000 + +RUN mkdir /addons \ + && groupadd -r -g ${GID} odoo \ + && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ + # && chown odoo /etc/odoo/odoo.conf \ + && chown -R odoo:odoo /addons \ + && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /usr/local/bin/odootools" >> /etc/sudoers + +# VOLUME /etc/odoo +VOLUME /var/lib/odoo + +ENV ODOO_RC /var/lib/odoo/odoo.conf +ENV ODOO_BASE_PATH /usr/local/lib/python3.8/dist-packages/odoo/addons +# Env variable defined to monitor the kind of service running +# it could be a staging/production/test or anything and undefined +# is the default in case we need to know servers that aren't correctly +# defined +ENV DEPLOYMENT_AREA undefined + +expose 8069 +expose 8071 + +USER odoo + +LABEL version="15.0" +LABEL release="20221104" + +LABEL org.opencontainers.image.created="2022-11-04T20:06:13.805177" +LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" +LABEL org.opencontainers.image.authors="OdooPlus " +LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" +LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" +LABEL org.opencontainers.image.version="15.0" +LABEL org.opencontainers.image.vendor="OdooPlus" +LABEL org.opencontainers.image.ref.name="15.0" +LABEL org.opencontainers.image.title="Odoo 15.0" +LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." + +ENTRYPOINT ["odootools", "--log-level", "INFO", "entrypoint", "user"] + +cmd ["odoo"] diff --git a/14.3/entrypoint.py b/15.0/entrypoint.py similarity index 69% rename from 14.3/entrypoint.py rename to 15.0/entrypoint.py index f2ce4654b..bb0ffddc3 100755 --- a/14.3/entrypoint.py +++ b/15.0/entrypoint.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # import argparse +from __future__ import print_function import time import shlex import subprocess @@ -17,25 +18,9 @@ from passlib.context import CryptContext from contextlib import contextmanager -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - -try: - from pip._internal.network.session import PipSession - from pip._internal.req.req_file import parse_requirements - from pip._internal.req.constructors import ( - install_req_from_parsed_requirement - ) -except Exception: - from pip.download import PipSession - from pip.req.req_file import parse_requirements - - def install_req_from_parsed_requirement(req): - return req - -from collections import defaultdict +from odoo_tools.compat import Path +from odoo_tools.modules.search import find_addons_paths, find_modules_paths +from odoo_tools.configuration.pips import merge_requirements try: # python3 @@ -68,63 +53,6 @@ def quote(s): return "'" + s.replace("'", "'\"'\"'") + "'" -class Requirement(object): - def __init__(self): - self.extras = set() - self.specifiers = set() - self.links = set() - self.editable = False - - -def merge_requirements(files): - requirements = defaultdict(lambda: Requirement()) - links = set() - - for filename in files: - f_requirements = parse_requirements( - str(filename), session=PipSession() - ) - for parsed_requirement in f_requirements: - requirement = install_req_from_parsed_requirement( - parsed_requirement - ) - - if requirement.markers and not requirement.markers.evaluate(): - continue - - if not hasattr(requirement.req, 'name'): - links.add(requirement.link.url) - break - name = requirement.req.name - specifiers = requirement.req.specifier - extras = requirement.req.extras - requirements[name].extras |= set(extras) - requirements[name].specifiers |= set(specifiers) - if requirement.link: - requirements[name].links |= {requirement.link.url} - requirements[name].editable |= requirement.editable - - result = [] - for key, value in requirements.items(): - if value.links: - result.append("%s" % value.links.pop()) - elif not value.extras: - result.append( - "%s %s" % (key, ",".join(map(str, value.specifiers))) - ) - else: - result.append("%s [%s] %s" % ( - key, - ",".join(map(str, value.extras)), - ",".join(map(str, value.specifiers)) - )) - - for link in links: - result.append(link) - - return "\n".join(result) - - def pipe(args): """ Call the process with std(in,out,err) @@ -178,42 +106,12 @@ def call_sudo_entrypoint(): return ret -def get_all_manifests(): - odoo_manifests = glob.glob('/addons/**/__manifest__.py', recursive=True) - erp_manifest = glob.glob('/addons/**/__openerp__.py', recursive=True) - - return odoo_manifests + erp_manifest - - -def module_name(manifest_file): - return path.basename(path.dirname(manifest_file)) - - -def get_module(manifest_file): - with open(manifest_file) as manifest_in: - try: - code = compile(manifest_in.read(), '__manifest__', 'eval') - module = eval(code, {}, {}) - except Exception: - module = {} - - return module - - -def is_installable(manifest): - return manifest.get('installable', True) - - -def is_server_wide(manifest): - return manifest.get('server_wide', False) - - def get_server_wide_modules(manifests): base_server_wide_modules = ['base', 'web'] custom_server_wide_modules = [ - module_name(filename) - for filename, module in manifests.items() - if is_server_wide(module) + manifest.technical_name + for manifest in manifests + if manifest.server_wide ] return base_server_wide_modules + custom_server_wide_modules @@ -246,7 +144,12 @@ def install_python_dependencies(valid_paths): req_file = Path('/var/lib/odoo/requirements.txt') with req_file.open('w') as fout: data = merge_requirements(requirement_files) - fout.write(data) + data = "\n".join(data) + + try: + fout.write(data) + except Exception: + fout.write(data.decode('utf-8')) for requirements in requirement_files: print("Installing python packages from %s" % requirements) @@ -338,65 +241,44 @@ def get_extra_paths(): ] -def get_valid_paths(): - base_addons = os.environ.get('ODOO_BASE_PATH') +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] - # addons = os.listdir('/addons') - valid_paths = [base_addons] +def get_valid_paths(): + base_addons = os.environ['ODOO_BASE_PATH'] - addons_paths = get_dirs('/addons') - addons_paths += get_extra_paths() + base_addons_paths = [base_addons, '/addons'] + base_addons_paths += get_extra_paths() - for addons_path in addons_paths: - print("Lookup addons in %s" % addons_path) - flush_streams() - addons = get_dirs(addons_path) - for addon in addons: - files = os.listdir(addon) - if ( - '__init__.py' in files and - '__manifest__.py' in files or - '__openerp__.py' in files - ): - valid_paths.append(addons_path) - print("Addons found !") - flush_streams() - break - if addons_path in valid_paths: - continue - else: - print("No addons found in path. Skipping...") - flush_streams() + valid_paths = find_addons_paths(base_addons_paths) + + for valid_path in valid_paths: + print("Found installable modules in {}".format(valid_path)) return valid_paths def setup_addons_paths(config, valid_paths): - config.set('options', 'addons_path', ",".join(valid_paths)) + config.set('options', 'addons_path', ",".join([ + str(path) for path in valid_paths + ])) flush_streams() -def setup_server_wide_modules(config): +def setup_server_wide_modules(config, valid_paths): print("Searching for server wide modules") - all_manifests = get_all_manifests() + filters = set(['installable']) + all_manifests = find_modules_paths(valid_paths, filters) - print("Found %s modules" % (len(all_manifests))) + print("Found %s installable modules " % (len(all_manifests),)) - manifests = { - filename: get_module(filename) - for filename in all_manifests - } - - installable_manifests = { - filename: module - for filename, module in manifests.items() - if is_installable(module) - } - - print("Found %s installable modules " % (len(installable_manifests),)) - - server_wide_modules = get_server_wide_modules(installable_manifests) + server_wide_modules = get_server_wide_modules(all_manifests) if len(server_wide_modules) > 2: modules = ",".join(server_wide_modules) @@ -408,6 +290,36 @@ def setup_server_wide_modules(config): flush_streams() +def convert_value(name, value): + if value in ['True', 'False']: + return value == 'True' + + return value + + +def setup_env_config(config): + try: + from odoo.tools import config as odoo_config + except ImportError: + from openerp.tools import config as odoo_config + + params_by_name = {} + for key, opt in odoo_config.casts.items(): + value_opt = opt.get_opt_string() + value_opt = value_opt.upper().replace('--', 'ODOO_') + value_opt = value_opt.replace('-', '_') + params_by_name[value_opt] = key + + for key, value in os.environ.items(): + if not key.startswith('ODOO_'): + continue + + if key in params_by_name: + config_name = params_by_name[key] + converted_value = convert_value(key, value) + config.set('options', config_name, converted_value) + + def setup_environ(config): print("Configuring environment variables for postgresql") @@ -499,12 +411,39 @@ def wait_postgresql(): @contextmanager def get_config(config_path): config = ConfigParser() - config.read(config_path) + config.read(str(config_path)) yield config - with config_path.open('w') as out: - config.write(out) + try: + with config_path.open('w') as out: + config.write(out) + except Exception: + with config_path.open('wb') as out: + config.write(out) + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for vpath in paths: + cur_path = Path(vpath) + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(vpath) + + return res_paths def main(): @@ -518,7 +457,9 @@ def main(): sys.exit(ret) # Install python packages with pip in user home - valid_paths = get_valid_paths() + orig_valid_paths = get_valid_paths() + excluded_paths = get_excluded_paths() + valid_paths = filter_valid_paths(orig_valid_paths, excluded_paths) config_path = Path(os.environ.get('ODOO_RC', '/etc/odoo/odoo.cfg')) @@ -528,7 +469,8 @@ def main(): install_master_password(config) setup_environ(config) setup_addons_paths(config, valid_paths) - setup_server_wide_modules(config) + setup_server_wide_modules(config, valid_paths) + setup_env_config(config) if not os.environ.get('ODOO_SKIP_POSTGRES_WAIT'): wait_postgresql() diff --git a/14.3/odoo.conf b/15.0/odoo.conf similarity index 100% rename from 14.3/odoo.conf rename to 15.0/odoo.conf diff --git a/15.0/platform.sh b/15.0/platform.sh new file mode 100644 index 000000000..fdf5e42eb --- /dev/null +++ b/15.0/platform.sh @@ -0,0 +1,13 @@ +PLATFORM=`uname -m` + +if [ $PLATFORM = "aarch64" ] +then + echo -n "arm64" +else +if [ $PLATFORM = "x86_64" ] +then + echo -n "amd64" +else + echo -n $PLATFORM +fi +fi diff --git a/14.3/sudo-entrypoint.py b/15.0/sudo-entrypoint.py similarity index 77% rename from 14.3/sudo-entrypoint.py rename to 15.0/sudo-entrypoint.py index 98779baae..916239170 100755 --- a/14.3/sudo-entrypoint.py +++ b/15.0/sudo-entrypoint.py @@ -12,6 +12,7 @@ from os import path from os.path import expanduser import shutil +import six try: from pathlib import Path @@ -69,6 +70,37 @@ def get_addons_paths(): ] +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for cur_path in paths: + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(cur_path) + + return res_paths + + def install_apt_packages(): """ Install debian dependencies. @@ -76,26 +108,34 @@ def install_apt_packages(): package_list = set() paths = get_addons_paths() + excluded_paths = get_excluded_paths() + paths = filter_valid_paths(paths, excluded_paths) print("Looking up for packages in {}".format(paths)) for addons_path in paths: for packages in addons_path.glob('**/apt-packages.txt'): print("Installing packages from %s" % packages) - with open(packages, 'r') as pack_file: - lines = [line.strip() for line in pack_file] + with packages.open('r') as pack_file: + lines = [ + six.ensure_text(line).strip() + for line in pack_file + if six.ensure_text(line).strip() + ] package_list.update(set(lines)) extras = os.environ.get('EXTRA_APT_PACKAGES', '') - print(f"Adding extra packages {extras}") + print("Adding extra packages {extras}".format(extras=extras)) if extras: for package in extras.split(','): - if not package: + if not package.strip(): continue - package_list.add(package) + package_list.add( + six.ensure_text(package).strip() + ) if len(package_list) > 0: - print(f"Installing {package_list}") + print("Installing {package_list}".format(package_list=package_list)) ret = pipe(['apt-get', 'update']) # Something went wrong, stop the service as it's failing diff --git a/16.0-nightly/Dockerfile b/16.0-nightly/Dockerfile new file mode 100644 index 000000000..6c73cc7b9 --- /dev/null +++ b/16.0-nightly/Dockerfile @@ -0,0 +1,138 @@ +FROM ubuntu:focal +LABEL maintainer="Archeti " + +# Generate locale C.UTF-8 for postgres and general locale data +ENV LANG C.UTF-8 +ARG DEBIAN_FRONTEND=noninteractive + +COPY ./platform.sh /platform.sh +# Install some dependencies python3.7 +RUN set -x; \ + export ARCH=`sh /platform.sh` \ + # Setup deb source for postgresql client + && apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg \ + python3.8 \ + python3-wheel \ + python3-setuptools \ + python3-pip \ + python3-psycopg2 \ + python3-ldap \ + python3-libsass \ + python3-lxml \ + python3-pillow \ + python3-pypdf2 \ + python3-psutil \ + python3-asn1crypto \ + curl \ + # libpq-dev \ + # libsasl2-2 \ + # libldap-2.4-2 \ + # libxml2 \ + # libxmlsec1 \ + # libxslt1.1 \ + sudo \ + node-less \ + && echo 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' > etc/apt/sources.list.d/pgdg.list \ + && export GNUPGHOME="$(mktemp -d)" \ + && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ + && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ + && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ + && gpgconf --kill all \ + && rm -rf "$GNUPGHOME" \ + # Download deb for wkhtmltox + && echo "Downloading https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_${ARCH}.deb" \ + && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_${ARCH}.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + ./wkhtmltox.deb \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1 \ + && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 2 \ + && rm -rf /var/lib/apt/lists/* wkhtmltox.deb + +ENV PATH=/var/lib/odoo/.local/bin:/usr/local/bin:$PATH +# Install Odoo Including things from sources +ENV ODOO_VERSION 16.0 +ENV ODOO_RELEASE= +ARG ODOO_ARCHIVE=odoo_${ODOO_VERSION}.${ODOO_RELEASE}.tar.gz +RUN set -x; \ + apt-get update \ + && apt-get install -y --no-install-recommends \ + # build-essential \ + # python3.8-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ + && pip install -U pip \ + && /usr/bin/env pip install \ + odoo-tools \ + \ + && odootools manage setup --release "" --repo "https://github.com/odoo/odoo.git" --ref "" "16.0" \ + && cd / \ + && apt-get --purge remove -y \ + # build-essential \ + # python3.8-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /root/.cache + +# COPY ./odoo.conf /etc/odoo/ +# COPY ./entrypoint.py / +# COPY ./sudo-entrypoint.py / + +ARG UID=1000 +ARG GID=1000 + +RUN mkdir /addons \ + && groupadd -r -g ${GID} odoo \ + && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ + # && chown odoo /etc/odoo/odoo.conf \ + && chown -R odoo:odoo /addons \ + && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /usr/local/bin/odootools" >> /etc/sudoers + +# VOLUME /etc/odoo +VOLUME /var/lib/odoo + +ENV ODOO_RC /var/lib/odoo/odoo.conf +ENV ODOO_BASE_PATH /usr/local/lib/python3.8/dist-packages/odoo/addons +# Env variable defined to monitor the kind of service running +# it could be a staging/production/test or anything and undefined +# is the default in case we need to know servers that aren't correctly +# defined +ENV DEPLOYMENT_AREA undefined + +expose 8069 +expose 8071 + +USER odoo + +LABEL version="16.0" +LABEL release="" + +LABEL org.opencontainers.image.created="2022-11-04T20:06:13.826902" +LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" +LABEL org.opencontainers.image.authors="OdooPlus " +LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" +LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" +LABEL org.opencontainers.image.version="16.0" +LABEL org.opencontainers.image.vendor="OdooPlus" +LABEL org.opencontainers.image.ref.name="16.0-nightly" +LABEL org.opencontainers.image.title="Odoo 16.0" +LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." + +ENTRYPOINT ["odootools", "--log-level", "INFO", "entrypoint", "user"] + +cmd ["odoo"] diff --git a/16.0-nightly/entrypoint.py b/16.0-nightly/entrypoint.py new file mode 100755 index 000000000..bb0ffddc3 --- /dev/null +++ b/16.0-nightly/entrypoint.py @@ -0,0 +1,501 @@ +#!/usr/bin/env python +# import argparse +from __future__ import print_function +import time +import shlex +import subprocess +import sys +import glob +# import pip +import re +import string +import random + +import os +from os import path +# from os.path import expanduser +import signal +from passlib.context import CryptContext +from contextlib import contextmanager + +from odoo_tools.compat import Path +from odoo_tools.modules.search import find_addons_paths, find_modules_paths +from odoo_tools.configuration.pips import merge_requirements + +try: + # python3 + SIGSEGV = signal.SIGSEGV.value +except AttributeError: + SIGSEGV = signal.SIGSEGV + + +try: + # python3 + from configparser import ConfigParser, NoOptionError +except Exception: + from ConfigParser import ConfigParser, NoOptionError + + +try: + # python3 + quote = shlex.quote +except Exception: + def quote(s): + """Return a shell-escaped version of the string *s*.""" + _find_unsafe = re.compile(r'[^\w@%+=:,./-]').search + if not s: + return "''" + if _find_unsafe(s) is None: + return s + + # use single quotes, and put single quotes into double quotes + # the string $'b is then quoted as '$'"'"'b' + return "'" + s.replace("'", "'\"'\"'") + "'" + + +def pipe(args): + """ + Call the process with std(in,out,err) + """ + print("Executing external command %s" % " ".join(args)) + flush_streams() + + env = os.environ.copy() + + process = subprocess.Popen( + args, + stdin=sys.stdin, + stdout=sys.stdout, + stderr=sys.stderr, + env=env + ) + + process.wait() + + print( + ( + "External command execution completed with returncode(%s)" + ) % process.returncode + ) + + if process.returncode == -SIGSEGV: + print("PIPE call segfaulted") + print("Failed to execute %s" % args) + + # Force a flush of buffer + flush_streams() + + return process.returncode + + +def start(): + """ + Main process running odoo + """ + print("Starting main command", sys.argv) + + return pipe(sys.argv[1:]) + + +def call_sudo_entrypoint(): + command = ["sudo", "-H", "-E"] + args = ["/sudo-entrypoint.py"] + + ret = pipe(command + args) + + return ret + + +def get_server_wide_modules(manifests): + base_server_wide_modules = ['base', 'web'] + custom_server_wide_modules = [ + manifest.technical_name + for manifest in manifests + if manifest.server_wide + ] + return base_server_wide_modules + custom_server_wide_modules + + +def install_python_dependencies(valid_paths): + """ + Install all the requirements.txt file found + """ + # TODO + # https://pypi.org/project/requirements-parser/ + # to parse all the requirements file to parse all the possible specs + # then append the specs to the loaded requirements and dump + # the requirements.txt file in /var/lib/odoo/requirements.txt and + # then install this only file instead of calling multiple time pip + # all_paths = ['/addons'] + get_extra_paths() + + requirement_files = [] + + for addons_path in valid_paths: + cur_path = Path(addons_path) + requirement_files += cur_path.glob("**/requirements.txt") + + requirement_files = list(set(requirement_files)) + requirement_files.sort() + + print("Installing python requirements found in:") + for f_path in requirement_files: + print(" {}".format(str(f_path))) + + req_file = Path('/var/lib/odoo/requirements.txt') + with req_file.open('w') as fout: + data = merge_requirements(requirement_files) + data = "\n".join(data) + + try: + fout.write(data) + except Exception: + fout.write(data.decode('utf-8')) + + for requirements in requirement_files: + print("Installing python packages from %s" % requirements) + flush_streams() + + print(data) + flush_streams() + + os.environ['PATH'] = "/var/lib/odoo/.local/bin:%s" % (os.environ['PATH'],) + retcode = pipe(["pip", "install", "--user", "-r", str(req_file)]) + flush_streams() + + if os.environ.get('ODOO_STRICT_MODE') and retcode != 0: + raise Exception("Failed to install pip dependencies") + + print("Installing python requirements complete\n") + flush_streams() + + +def randomString(stringLength=10): + """Generate a random string of fixed length """ + letters = string.ascii_lowercase + return ''.join(random.choice(letters) for i in range(stringLength)) + + +def install_master_password(config): + # Secure an odoo instance with a default master password + # if required we can update the master password but at least + # odoo doesn't get exposed by default without master passwords + print("Installing master password in ODOORC") + + ctx = CryptContext( + ['pbkdf2_sha512', 'plaintext'], + deprecated=['plaintext'] + ) + + master_password_secret = "/run/secrets/master_password" + if path.exists(master_password_secret): + with open(master_password_secret, "r") as mp: + master_password = mp.read().strip() + elif os.environ.get('MASTER_PASSWORD'): + master_password = os.environ.get('MASTER_PASSWORD') + else: + master_password = randomString(64) + + if os.environ.get('DEPLOYMENT_AREA') == 'undefined': + print( + "Use this randomly generated master password" + " to manage the database" + ) + print(" %s" % master_password) + + # Check that we don't have plaintext and encrypt it + # This allow us to quickly setup servers without having to hash + # ourselves first for security reason, you should always hash + # the password first and not expect the image to do it correctly + # but older version of odoo do not support encryption so only encrypt + # older version of odoo... + if ( + float(os.environ.get('ODOO_VERSION')) > 10 and + ctx.identify(master_password) == 'plaintext' + ): + master_password = ctx.encrypt(master_password) + + config.set('options', 'admin_passwd', master_password) + + print("Installing master password completed") + + flush_streams() + + +def get_dirs(cur_path): + return [ + path.join(cur_path, npath) + for npath in os.listdir(cur_path) + if path.isdir(path.join(cur_path, npath)) + ] + + +def get_extra_paths(): + extra_paths = os.environ.get('ODOO_EXTRA_PATHS') + + if not extra_paths: + return [] + + return [ + extra_path.strip() + for extra_path in extra_paths.split(',') + ] + + +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] + + +def get_valid_paths(): + base_addons = os.environ['ODOO_BASE_PATH'] + + base_addons_paths = [base_addons, '/addons'] + base_addons_paths += get_extra_paths() + + valid_paths = find_addons_paths(base_addons_paths) + + for valid_path in valid_paths: + print("Found installable modules in {}".format(valid_path)) + + return valid_paths + + +def setup_addons_paths(config, valid_paths): + config.set('options', 'addons_path', ",".join([ + str(path) for path in valid_paths + ])) + flush_streams() + + +def setup_server_wide_modules(config, valid_paths): + print("Searching for server wide modules") + filters = set(['installable']) + all_manifests = find_modules_paths(valid_paths, filters) + + print("Found %s installable modules " % (len(all_manifests),)) + + server_wide_modules = get_server_wide_modules(all_manifests) + + if len(server_wide_modules) > 2: + modules = ",".join(server_wide_modules) + print("Setting server wide modules to %s" % (modules)) + config.set('options', 'server_wide_modules', modules) + else: + print("No server wide modules found") + + flush_streams() + + +def convert_value(name, value): + if value in ['True', 'False']: + return value == 'True' + + return value + + +def setup_env_config(config): + try: + from odoo.tools import config as odoo_config + except ImportError: + from openerp.tools import config as odoo_config + + params_by_name = {} + for key, opt in odoo_config.casts.items(): + value_opt = opt.get_opt_string() + value_opt = value_opt.upper().replace('--', 'ODOO_') + value_opt = value_opt.replace('-', '_') + params_by_name[value_opt] = key + + for key, value in os.environ.items(): + if not key.startswith('ODOO_'): + continue + + if key in params_by_name: + config_name = params_by_name[key] + converted_value = convert_value(key, value) + config.set('options', config_name, converted_value) + + +def setup_environ(config): + print("Configuring environment variables for postgresql") + + def get_option(config, section, name): + try: + return config.get(section, name) + except NoOptionError: + return None + + def check_config(config_name, config_small): + """ + Check if config is in odoo_rc or command line + """ + value = None + + if get_option(config, 'options', config_name): + value = get_option(config, 'options', config_name) + + if not value and '--%s' % config_name in sys.argv: + idx = sys.argv.index('--%s' % config_name) + value = sys.argv[idx + 1] if idx < len(sys.argv) else None + + if not value and config_small and '-%s' % config_small in sys.argv: + idx = sys.argv.index('-%s' % config_small) + value = sys.argv[idx + 1] if idx < len(sys.argv) else None + + return value + + variables = [ + ('PGUSER', 'db_user', 'r'), + ('PGHOST', 'db_host', None), + ('PGPORT', 'db_port', None), + ('PGDATABASE', 'database', 'd') + ] + + # Accpet db_password only with this if some infra cannot be setup + # otherwise... + # It's a bad idea to pass password in cleartext in command line or + # environment variables so please use .pgpass instead... + if os.environ.get('I_KNOW_WHAT_IM_DOING') == 'TRUE': + variables.append( + ('PGPASSWORD', 'db_password', 'w') + ) + + # Setup basic PG env variables to simplify managements + # combined with secret pg pass we can use psql directly + for pg_val, odoo_val, small_arg in variables: + value = check_config(odoo_val, small_arg) + if value: + os.environ[pg_val] = value + + print("Configuring environment variables done") + flush_streams() + + +def wait_postgresql(): + import psycopg2 + + retries = int(os.environ.get('PGRETRY', 5)) + retries_wait = int(os.environ.get('PGRETRYTIME', 1)) + error = None + + # Default database set to postgres + if not os.environ.get('PGDATABASE'): + os.environ['PGDATABASE'] = 'postgres' + + for retry in range(retries): + try: + print("Trying to connect to postgresql") + # connect using defined env variables and pgpass files + flush_streams() + conn = psycopg2.connect("") + message = " Connected to %(user)s@%(host)s:%(port)s" + print(message % conn.get_dsn_parameters()) + flush_streams() + break + except psycopg2.OperationalError as exc: + error = exc + time.sleep(retries_wait) + else: + # we reached the maximum retries so we trigger failure mode + if error: + print("Database connection failure %s" % error) + flush_streams() + + sys.exit(1) + + +@contextmanager +def get_config(config_path): + config = ConfigParser() + config.read(str(config_path)) + + yield config + + try: + with config_path.open('w') as out: + config.write(out) + except Exception: + with config_path.open('wb') as out: + config.write(out) + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for vpath in paths: + cur_path = Path(vpath) + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(vpath) + + return res_paths + + +def main(): + # Install apt package first then python packages + if not os.environ.get('SKIP_SUDO_ENTRYPOINT'): + ret = call_sudo_entrypoint() + else: + ret = 0 + + if ret not in [0, None]: + sys.exit(ret) + + # Install python packages with pip in user home + orig_valid_paths = get_valid_paths() + excluded_paths = get_excluded_paths() + valid_paths = filter_valid_paths(orig_valid_paths, excluded_paths) + + config_path = Path(os.environ.get('ODOO_RC', '/etc/odoo/odoo.cfg')) + + with get_config(config_path) as config: + if not os.environ.get('SKIP_PIP'): + install_python_dependencies(valid_paths) + install_master_password(config) + setup_environ(config) + setup_addons_paths(config, valid_paths) + setup_server_wide_modules(config, valid_paths) + setup_env_config(config) + + if not os.environ.get('ODOO_SKIP_POSTGRES_WAIT'): + wait_postgresql() + + return start() + + +def flush_streams(): + sys.stdout.flush() + sys.stderr.flush() + + +try: + code = main() + flush_streams() + sys.exit(code) +except Exception as exc: + print(exc) + import traceback + traceback.print_exc() + flush_streams() + sys.exit(1) +except KeyboardInterrupt as exc: + print(exc) + import traceback + traceback.print_exc() + flush_streams() + sys.exit(1) diff --git a/16.0-nightly/odoo.conf b/16.0-nightly/odoo.conf new file mode 100644 index 000000000..28f70c105 --- /dev/null +++ b/16.0-nightly/odoo.conf @@ -0,0 +1,37 @@ +[options] +addons_path = /mnt/extra-addons +data_dir = /var/lib/odoo +; admin_passwd = admin +; csv_internal_sep = , +; db_maxconn = 64 +; db_name = False +; db_template = template1 +; dbfilter = .* +; debug_mode = False +; email_from = False +; limit_memory_hard = 2684354560 +; limit_memory_soft = 2147483648 +; limit_request = 8192 +; limit_time_cpu = 60 +; limit_time_real = 120 +; list_db = True +; log_db = False +; log_handler = [':INFO'] +; log_level = info +; logfile = None +; longpolling_port = 8072 +; max_cron_threads = 2 +; osv_memory_age_limit = 1.0 +; osv_memory_count_limit = False +; smtp_password = False +; smtp_port = 25 +; smtp_server = localhost +; smtp_ssl = False +; smtp_user = False +; workers = 0 +; xmlrpc = True +; xmlrpc_interface = +; xmlrpc_port = 8069 +; xmlrpcs = True +; xmlrpcs_interface = +; xmlrpcs_port = 8071 diff --git a/16.0-nightly/platform.sh b/16.0-nightly/platform.sh new file mode 100644 index 000000000..fdf5e42eb --- /dev/null +++ b/16.0-nightly/platform.sh @@ -0,0 +1,13 @@ +PLATFORM=`uname -m` + +if [ $PLATFORM = "aarch64" ] +then + echo -n "arm64" +else +if [ $PLATFORM = "x86_64" ] +then + echo -n "amd64" +else + echo -n $PLATFORM +fi +fi diff --git a/16.0-nightly/sudo-entrypoint.py b/16.0-nightly/sudo-entrypoint.py new file mode 100755 index 000000000..916239170 --- /dev/null +++ b/16.0-nightly/sudo-entrypoint.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python +import argparse +import time +import os +import shlex +import subprocess +import sys +import glob +import pip +import re +import stat +from os import path +from os.path import expanduser +import shutil +import six + +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + + +def pipe(args): + """ + Call the process with std(in,out,err) + """ + env = os.environ.copy() + env['DEBIAN_FRONTEND'] = 'noninteractive' + + process = subprocess.Popen( + args, + stdin=sys.stdin, + stdout=sys.stdout, + stderr=sys.stderr, + env=env + ) + + process.wait() + + return process.returncode + + +def get_dirs(cur_path): + return [ + path.join(cur_path, npath) + for npath in os.listdir(cur_path) + if path.isdir(path.join(cur_path, npath)) + ] + + +def get_extra_paths(): + extra_paths = os.environ.get('ODOO_EXTRA_PATHS') + + if not extra_paths: + return [] + + return [ + extra_path.strip() + for extra_path in extra_paths.split(',') + ] + + +def get_addons_paths(): + addons = get_dirs('/addons') + addons += get_extra_paths() + + return [ + Path(path) + for path in addons + ] + + +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for cur_path in paths: + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(cur_path) + + return res_paths + + +def install_apt_packages(): + """ + Install debian dependencies. + """ + package_list = set() + + paths = get_addons_paths() + excluded_paths = get_excluded_paths() + paths = filter_valid_paths(paths, excluded_paths) + + print("Looking up for packages in {}".format(paths)) + + for addons_path in paths: + for packages in addons_path.glob('**/apt-packages.txt'): + print("Installing packages from %s" % packages) + with packages.open('r') as pack_file: + lines = [ + six.ensure_text(line).strip() + for line in pack_file + if six.ensure_text(line).strip() + ] + package_list.update(set(lines)) + + extras = os.environ.get('EXTRA_APT_PACKAGES', '') + print("Adding extra packages {extras}".format(extras=extras)) + if extras: + for package in extras.split(','): + if not package.strip(): + continue + package_list.add( + six.ensure_text(package).strip() + ) + + if len(package_list) > 0: + print("Installing {package_list}".format(package_list=package_list)) + ret = pipe(['apt-get', 'update']) + + # Something went wrong, stop the service as it's failing + if ret != 0: + sys.exit(ret) + + ret = pipe(['apt-get', 'install', '-y'] + list(package_list)) + + # Something went wrong, stop the service as it's failing + if ret != 0: + sys.exit(ret) + + +def load_secrets(): + # TODO add a way to load some secrets so odoo process can + # use secrets as a way to load passwords/user for postgresql + # credentials could also be stored in the HOME of the odoo user + # except we cannot rely on secrets 100% because it only works in + # swarm mode + pgpass_secret = '/run/secrets/.pgpass' + if path.exists(pgpass_secret): + home_folder = '/var/lib/odoo' + pgpass_target = path.join(home_folder, '.pgpass') + if path.exists(pgpass_target): + os.remove(pgpass_target) + # shutil.move doesn't always work correctly on different fs + shutil.copy(pgpass_secret, home_folder) + st = os.stat(pgpass_secret) + os.chmod(pgpass_target, st.st_mode) + os.chown(pgpass_target, st[stat.ST_UID], st[stat.ST_GID]) + # Cannot remove anymore apparently + # os.remove(pgpass_secret) + # shutil.move(pgpass_secret, home_folder) + + +def disable_base_modules(): + base_addons = os.environ.get('ODOO_BASE_PATH', '') + addons_to_remove = os.environ.get('ODOO_DISABLED_MODULES', '') + + modules = addons_to_remove.split(',') + modules = map(lambda mod: mod.strip(), modules) + + if not base_addons: + print("Do not attempt to remove wrong folder") + return + + for module in modules: + if not module: + continue + print("Removing module %s from %s" % (module, base_addons)) + + module_path = Path(base_addons, module) + if module_path.exists() and module_path.is_dir(): + shutil.rmtree(module_path) + else: + print("Module skipped as it doesn't seem to be present.") + + +def fix_access_rights(): + if os.environ.get('RESET_ACCESS_RIGHTS', '') == 'TRUE': + pipe(["chown", "-R", "odoo:odoo", "/var/lib/odoo"]) + pipe(["chown", "-R", "odoo:odoo", "/etc/odoo"]) + + +def remove_sudo(): + return pipe(["sed", "-i", "/odoo/d", "/etc/sudoers"]) + + +def main(): + install_apt_packages() + load_secrets() + fix_access_rights() + disable_base_modules() + return remove_sudo() + + +try: + code = main() + sys.exit(code) +except Exception as exc: + print(exc) + sys.exit(1) +except KeyboardInterrupt as exc: + print(exc) + sys.exit(1) diff --git a/16.0/Dockerfile b/16.0/Dockerfile new file mode 100644 index 000000000..04d58e71e --- /dev/null +++ b/16.0/Dockerfile @@ -0,0 +1,138 @@ +FROM ubuntu:focal +LABEL maintainer="Archeti " + +# Generate locale C.UTF-8 for postgres and general locale data +ENV LANG C.UTF-8 +ARG DEBIAN_FRONTEND=noninteractive + +COPY ./platform.sh /platform.sh +# Install some dependencies python3.7 +RUN set -x; \ + export ARCH=`sh /platform.sh` \ + # Setup deb source for postgresql client + && apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg \ + python3.8 \ + python3-wheel \ + python3-setuptools \ + python3-pip \ + python3-psycopg2 \ + python3-ldap \ + python3-libsass \ + python3-lxml \ + python3-pillow \ + python3-pypdf2 \ + python3-psutil \ + python3-asn1crypto \ + curl \ + # libpq-dev \ + # libsasl2-2 \ + # libldap-2.4-2 \ + # libxml2 \ + # libxmlsec1 \ + # libxslt1.1 \ + sudo \ + node-less \ + && echo 'deb https://apt.postgresql.org/pub/repos/apt/ focal-pgdg main' > etc/apt/sources.list.d/pgdg.list \ + && export GNUPGHOME="$(mktemp -d)" \ + && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ + && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ + && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ + && gpgconf --kill all \ + && rm -rf "$GNUPGHOME" \ + # Download deb for wkhtmltox + && echo "Downloading https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_${ARCH}.deb" \ + && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.focal_${ARCH}.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + ./wkhtmltox.deb \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1 \ + && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 2 \ + && rm -rf /var/lib/apt/lists/* wkhtmltox.deb + +ENV PATH=/var/lib/odoo/.local/bin:/usr/local/bin:$PATH +# Install Odoo Including things from sources +ENV ODOO_VERSION 16.0 +ENV ODOO_RELEASE=20221104 +ARG ODOO_ARCHIVE=odoo_${ODOO_VERSION}.${ODOO_RELEASE}.tar.gz +RUN set -x; \ + apt-get update \ + && apt-get install -y --no-install-recommends \ + # build-essential \ + # python3.8-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ + && pip install -U pip \ + && /usr/bin/env pip install \ + odoo-tools \ + \ + && odootools manage setup --release "20221104" --repo "https://github.com/odoo/odoo.git" --ref "" "16.0" \ + && cd / \ + && apt-get --purge remove -y \ + # build-essential \ + # python3.8-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ + git \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /root/.cache + +# COPY ./odoo.conf /etc/odoo/ +# COPY ./entrypoint.py / +# COPY ./sudo-entrypoint.py / + +ARG UID=1000 +ARG GID=1000 + +RUN mkdir /addons \ + && groupadd -r -g ${GID} odoo \ + && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ + # && chown odoo /etc/odoo/odoo.conf \ + && chown -R odoo:odoo /addons \ + && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /usr/local/bin/odootools" >> /etc/sudoers + +# VOLUME /etc/odoo +VOLUME /var/lib/odoo + +ENV ODOO_RC /var/lib/odoo/odoo.conf +ENV ODOO_BASE_PATH /usr/local/lib/python3.8/dist-packages/odoo/addons +# Env variable defined to monitor the kind of service running +# it could be a staging/production/test or anything and undefined +# is the default in case we need to know servers that aren't correctly +# defined +ENV DEPLOYMENT_AREA undefined + +expose 8069 +expose 8071 + +USER odoo + +LABEL version="16.0" +LABEL release="20221104" + +LABEL org.opencontainers.image.created="2022-11-04T20:06:13.806119" +LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" +LABEL org.opencontainers.image.authors="OdooPlus " +LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" +LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" +LABEL org.opencontainers.image.version="16.0" +LABEL org.opencontainers.image.vendor="OdooPlus" +LABEL org.opencontainers.image.ref.name="16.0" +LABEL org.opencontainers.image.title="Odoo 16.0" +LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." + +ENTRYPOINT ["odootools", "--log-level", "INFO", "entrypoint", "user"] + +cmd ["odoo"] diff --git a/16.0/entrypoint.py b/16.0/entrypoint.py new file mode 100755 index 000000000..bb0ffddc3 --- /dev/null +++ b/16.0/entrypoint.py @@ -0,0 +1,501 @@ +#!/usr/bin/env python +# import argparse +from __future__ import print_function +import time +import shlex +import subprocess +import sys +import glob +# import pip +import re +import string +import random + +import os +from os import path +# from os.path import expanduser +import signal +from passlib.context import CryptContext +from contextlib import contextmanager + +from odoo_tools.compat import Path +from odoo_tools.modules.search import find_addons_paths, find_modules_paths +from odoo_tools.configuration.pips import merge_requirements + +try: + # python3 + SIGSEGV = signal.SIGSEGV.value +except AttributeError: + SIGSEGV = signal.SIGSEGV + + +try: + # python3 + from configparser import ConfigParser, NoOptionError +except Exception: + from ConfigParser import ConfigParser, NoOptionError + + +try: + # python3 + quote = shlex.quote +except Exception: + def quote(s): + """Return a shell-escaped version of the string *s*.""" + _find_unsafe = re.compile(r'[^\w@%+=:,./-]').search + if not s: + return "''" + if _find_unsafe(s) is None: + return s + + # use single quotes, and put single quotes into double quotes + # the string $'b is then quoted as '$'"'"'b' + return "'" + s.replace("'", "'\"'\"'") + "'" + + +def pipe(args): + """ + Call the process with std(in,out,err) + """ + print("Executing external command %s" % " ".join(args)) + flush_streams() + + env = os.environ.copy() + + process = subprocess.Popen( + args, + stdin=sys.stdin, + stdout=sys.stdout, + stderr=sys.stderr, + env=env + ) + + process.wait() + + print( + ( + "External command execution completed with returncode(%s)" + ) % process.returncode + ) + + if process.returncode == -SIGSEGV: + print("PIPE call segfaulted") + print("Failed to execute %s" % args) + + # Force a flush of buffer + flush_streams() + + return process.returncode + + +def start(): + """ + Main process running odoo + """ + print("Starting main command", sys.argv) + + return pipe(sys.argv[1:]) + + +def call_sudo_entrypoint(): + command = ["sudo", "-H", "-E"] + args = ["/sudo-entrypoint.py"] + + ret = pipe(command + args) + + return ret + + +def get_server_wide_modules(manifests): + base_server_wide_modules = ['base', 'web'] + custom_server_wide_modules = [ + manifest.technical_name + for manifest in manifests + if manifest.server_wide + ] + return base_server_wide_modules + custom_server_wide_modules + + +def install_python_dependencies(valid_paths): + """ + Install all the requirements.txt file found + """ + # TODO + # https://pypi.org/project/requirements-parser/ + # to parse all the requirements file to parse all the possible specs + # then append the specs to the loaded requirements and dump + # the requirements.txt file in /var/lib/odoo/requirements.txt and + # then install this only file instead of calling multiple time pip + # all_paths = ['/addons'] + get_extra_paths() + + requirement_files = [] + + for addons_path in valid_paths: + cur_path = Path(addons_path) + requirement_files += cur_path.glob("**/requirements.txt") + + requirement_files = list(set(requirement_files)) + requirement_files.sort() + + print("Installing python requirements found in:") + for f_path in requirement_files: + print(" {}".format(str(f_path))) + + req_file = Path('/var/lib/odoo/requirements.txt') + with req_file.open('w') as fout: + data = merge_requirements(requirement_files) + data = "\n".join(data) + + try: + fout.write(data) + except Exception: + fout.write(data.decode('utf-8')) + + for requirements in requirement_files: + print("Installing python packages from %s" % requirements) + flush_streams() + + print(data) + flush_streams() + + os.environ['PATH'] = "/var/lib/odoo/.local/bin:%s" % (os.environ['PATH'],) + retcode = pipe(["pip", "install", "--user", "-r", str(req_file)]) + flush_streams() + + if os.environ.get('ODOO_STRICT_MODE') and retcode != 0: + raise Exception("Failed to install pip dependencies") + + print("Installing python requirements complete\n") + flush_streams() + + +def randomString(stringLength=10): + """Generate a random string of fixed length """ + letters = string.ascii_lowercase + return ''.join(random.choice(letters) for i in range(stringLength)) + + +def install_master_password(config): + # Secure an odoo instance with a default master password + # if required we can update the master password but at least + # odoo doesn't get exposed by default without master passwords + print("Installing master password in ODOORC") + + ctx = CryptContext( + ['pbkdf2_sha512', 'plaintext'], + deprecated=['plaintext'] + ) + + master_password_secret = "/run/secrets/master_password" + if path.exists(master_password_secret): + with open(master_password_secret, "r") as mp: + master_password = mp.read().strip() + elif os.environ.get('MASTER_PASSWORD'): + master_password = os.environ.get('MASTER_PASSWORD') + else: + master_password = randomString(64) + + if os.environ.get('DEPLOYMENT_AREA') == 'undefined': + print( + "Use this randomly generated master password" + " to manage the database" + ) + print(" %s" % master_password) + + # Check that we don't have plaintext and encrypt it + # This allow us to quickly setup servers without having to hash + # ourselves first for security reason, you should always hash + # the password first and not expect the image to do it correctly + # but older version of odoo do not support encryption so only encrypt + # older version of odoo... + if ( + float(os.environ.get('ODOO_VERSION')) > 10 and + ctx.identify(master_password) == 'plaintext' + ): + master_password = ctx.encrypt(master_password) + + config.set('options', 'admin_passwd', master_password) + + print("Installing master password completed") + + flush_streams() + + +def get_dirs(cur_path): + return [ + path.join(cur_path, npath) + for npath in os.listdir(cur_path) + if path.isdir(path.join(cur_path, npath)) + ] + + +def get_extra_paths(): + extra_paths = os.environ.get('ODOO_EXTRA_PATHS') + + if not extra_paths: + return [] + + return [ + extra_path.strip() + for extra_path in extra_paths.split(',') + ] + + +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] + + +def get_valid_paths(): + base_addons = os.environ['ODOO_BASE_PATH'] + + base_addons_paths = [base_addons, '/addons'] + base_addons_paths += get_extra_paths() + + valid_paths = find_addons_paths(base_addons_paths) + + for valid_path in valid_paths: + print("Found installable modules in {}".format(valid_path)) + + return valid_paths + + +def setup_addons_paths(config, valid_paths): + config.set('options', 'addons_path', ",".join([ + str(path) for path in valid_paths + ])) + flush_streams() + + +def setup_server_wide_modules(config, valid_paths): + print("Searching for server wide modules") + filters = set(['installable']) + all_manifests = find_modules_paths(valid_paths, filters) + + print("Found %s installable modules " % (len(all_manifests),)) + + server_wide_modules = get_server_wide_modules(all_manifests) + + if len(server_wide_modules) > 2: + modules = ",".join(server_wide_modules) + print("Setting server wide modules to %s" % (modules)) + config.set('options', 'server_wide_modules', modules) + else: + print("No server wide modules found") + + flush_streams() + + +def convert_value(name, value): + if value in ['True', 'False']: + return value == 'True' + + return value + + +def setup_env_config(config): + try: + from odoo.tools import config as odoo_config + except ImportError: + from openerp.tools import config as odoo_config + + params_by_name = {} + for key, opt in odoo_config.casts.items(): + value_opt = opt.get_opt_string() + value_opt = value_opt.upper().replace('--', 'ODOO_') + value_opt = value_opt.replace('-', '_') + params_by_name[value_opt] = key + + for key, value in os.environ.items(): + if not key.startswith('ODOO_'): + continue + + if key in params_by_name: + config_name = params_by_name[key] + converted_value = convert_value(key, value) + config.set('options', config_name, converted_value) + + +def setup_environ(config): + print("Configuring environment variables for postgresql") + + def get_option(config, section, name): + try: + return config.get(section, name) + except NoOptionError: + return None + + def check_config(config_name, config_small): + """ + Check if config is in odoo_rc or command line + """ + value = None + + if get_option(config, 'options', config_name): + value = get_option(config, 'options', config_name) + + if not value and '--%s' % config_name in sys.argv: + idx = sys.argv.index('--%s' % config_name) + value = sys.argv[idx + 1] if idx < len(sys.argv) else None + + if not value and config_small and '-%s' % config_small in sys.argv: + idx = sys.argv.index('-%s' % config_small) + value = sys.argv[idx + 1] if idx < len(sys.argv) else None + + return value + + variables = [ + ('PGUSER', 'db_user', 'r'), + ('PGHOST', 'db_host', None), + ('PGPORT', 'db_port', None), + ('PGDATABASE', 'database', 'd') + ] + + # Accpet db_password only with this if some infra cannot be setup + # otherwise... + # It's a bad idea to pass password in cleartext in command line or + # environment variables so please use .pgpass instead... + if os.environ.get('I_KNOW_WHAT_IM_DOING') == 'TRUE': + variables.append( + ('PGPASSWORD', 'db_password', 'w') + ) + + # Setup basic PG env variables to simplify managements + # combined with secret pg pass we can use psql directly + for pg_val, odoo_val, small_arg in variables: + value = check_config(odoo_val, small_arg) + if value: + os.environ[pg_val] = value + + print("Configuring environment variables done") + flush_streams() + + +def wait_postgresql(): + import psycopg2 + + retries = int(os.environ.get('PGRETRY', 5)) + retries_wait = int(os.environ.get('PGRETRYTIME', 1)) + error = None + + # Default database set to postgres + if not os.environ.get('PGDATABASE'): + os.environ['PGDATABASE'] = 'postgres' + + for retry in range(retries): + try: + print("Trying to connect to postgresql") + # connect using defined env variables and pgpass files + flush_streams() + conn = psycopg2.connect("") + message = " Connected to %(user)s@%(host)s:%(port)s" + print(message % conn.get_dsn_parameters()) + flush_streams() + break + except psycopg2.OperationalError as exc: + error = exc + time.sleep(retries_wait) + else: + # we reached the maximum retries so we trigger failure mode + if error: + print("Database connection failure %s" % error) + flush_streams() + + sys.exit(1) + + +@contextmanager +def get_config(config_path): + config = ConfigParser() + config.read(str(config_path)) + + yield config + + try: + with config_path.open('w') as out: + config.write(out) + except Exception: + with config_path.open('wb') as out: + config.write(out) + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for vpath in paths: + cur_path = Path(vpath) + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(vpath) + + return res_paths + + +def main(): + # Install apt package first then python packages + if not os.environ.get('SKIP_SUDO_ENTRYPOINT'): + ret = call_sudo_entrypoint() + else: + ret = 0 + + if ret not in [0, None]: + sys.exit(ret) + + # Install python packages with pip in user home + orig_valid_paths = get_valid_paths() + excluded_paths = get_excluded_paths() + valid_paths = filter_valid_paths(orig_valid_paths, excluded_paths) + + config_path = Path(os.environ.get('ODOO_RC', '/etc/odoo/odoo.cfg')) + + with get_config(config_path) as config: + if not os.environ.get('SKIP_PIP'): + install_python_dependencies(valid_paths) + install_master_password(config) + setup_environ(config) + setup_addons_paths(config, valid_paths) + setup_server_wide_modules(config, valid_paths) + setup_env_config(config) + + if not os.environ.get('ODOO_SKIP_POSTGRES_WAIT'): + wait_postgresql() + + return start() + + +def flush_streams(): + sys.stdout.flush() + sys.stderr.flush() + + +try: + code = main() + flush_streams() + sys.exit(code) +except Exception as exc: + print(exc) + import traceback + traceback.print_exc() + flush_streams() + sys.exit(1) +except KeyboardInterrupt as exc: + print(exc) + import traceback + traceback.print_exc() + flush_streams() + sys.exit(1) diff --git a/16.0/odoo.conf b/16.0/odoo.conf new file mode 100644 index 000000000..28f70c105 --- /dev/null +++ b/16.0/odoo.conf @@ -0,0 +1,37 @@ +[options] +addons_path = /mnt/extra-addons +data_dir = /var/lib/odoo +; admin_passwd = admin +; csv_internal_sep = , +; db_maxconn = 64 +; db_name = False +; db_template = template1 +; dbfilter = .* +; debug_mode = False +; email_from = False +; limit_memory_hard = 2684354560 +; limit_memory_soft = 2147483648 +; limit_request = 8192 +; limit_time_cpu = 60 +; limit_time_real = 120 +; list_db = True +; log_db = False +; log_handler = [':INFO'] +; log_level = info +; logfile = None +; longpolling_port = 8072 +; max_cron_threads = 2 +; osv_memory_age_limit = 1.0 +; osv_memory_count_limit = False +; smtp_password = False +; smtp_port = 25 +; smtp_server = localhost +; smtp_ssl = False +; smtp_user = False +; workers = 0 +; xmlrpc = True +; xmlrpc_interface = +; xmlrpc_port = 8069 +; xmlrpcs = True +; xmlrpcs_interface = +; xmlrpcs_port = 8071 diff --git a/16.0/platform.sh b/16.0/platform.sh new file mode 100644 index 000000000..fdf5e42eb --- /dev/null +++ b/16.0/platform.sh @@ -0,0 +1,13 @@ +PLATFORM=`uname -m` + +if [ $PLATFORM = "aarch64" ] +then + echo -n "arm64" +else +if [ $PLATFORM = "x86_64" ] +then + echo -n "amd64" +else + echo -n $PLATFORM +fi +fi diff --git a/16.0/sudo-entrypoint.py b/16.0/sudo-entrypoint.py new file mode 100755 index 000000000..916239170 --- /dev/null +++ b/16.0/sudo-entrypoint.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python +import argparse +import time +import os +import shlex +import subprocess +import sys +import glob +import pip +import re +import stat +from os import path +from os.path import expanduser +import shutil +import six + +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + + +def pipe(args): + """ + Call the process with std(in,out,err) + """ + env = os.environ.copy() + env['DEBIAN_FRONTEND'] = 'noninteractive' + + process = subprocess.Popen( + args, + stdin=sys.stdin, + stdout=sys.stdout, + stderr=sys.stderr, + env=env + ) + + process.wait() + + return process.returncode + + +def get_dirs(cur_path): + return [ + path.join(cur_path, npath) + for npath in os.listdir(cur_path) + if path.isdir(path.join(cur_path, npath)) + ] + + +def get_extra_paths(): + extra_paths = os.environ.get('ODOO_EXTRA_PATHS') + + if not extra_paths: + return [] + + return [ + extra_path.strip() + for extra_path in extra_paths.split(',') + ] + + +def get_addons_paths(): + addons = get_dirs('/addons') + addons += get_extra_paths() + + return [ + Path(path) + for path in addons + ] + + +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for cur_path in paths: + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(cur_path) + + return res_paths + + +def install_apt_packages(): + """ + Install debian dependencies. + """ + package_list = set() + + paths = get_addons_paths() + excluded_paths = get_excluded_paths() + paths = filter_valid_paths(paths, excluded_paths) + + print("Looking up for packages in {}".format(paths)) + + for addons_path in paths: + for packages in addons_path.glob('**/apt-packages.txt'): + print("Installing packages from %s" % packages) + with packages.open('r') as pack_file: + lines = [ + six.ensure_text(line).strip() + for line in pack_file + if six.ensure_text(line).strip() + ] + package_list.update(set(lines)) + + extras = os.environ.get('EXTRA_APT_PACKAGES', '') + print("Adding extra packages {extras}".format(extras=extras)) + if extras: + for package in extras.split(','): + if not package.strip(): + continue + package_list.add( + six.ensure_text(package).strip() + ) + + if len(package_list) > 0: + print("Installing {package_list}".format(package_list=package_list)) + ret = pipe(['apt-get', 'update']) + + # Something went wrong, stop the service as it's failing + if ret != 0: + sys.exit(ret) + + ret = pipe(['apt-get', 'install', '-y'] + list(package_list)) + + # Something went wrong, stop the service as it's failing + if ret != 0: + sys.exit(ret) + + +def load_secrets(): + # TODO add a way to load some secrets so odoo process can + # use secrets as a way to load passwords/user for postgresql + # credentials could also be stored in the HOME of the odoo user + # except we cannot rely on secrets 100% because it only works in + # swarm mode + pgpass_secret = '/run/secrets/.pgpass' + if path.exists(pgpass_secret): + home_folder = '/var/lib/odoo' + pgpass_target = path.join(home_folder, '.pgpass') + if path.exists(pgpass_target): + os.remove(pgpass_target) + # shutil.move doesn't always work correctly on different fs + shutil.copy(pgpass_secret, home_folder) + st = os.stat(pgpass_secret) + os.chmod(pgpass_target, st.st_mode) + os.chown(pgpass_target, st[stat.ST_UID], st[stat.ST_GID]) + # Cannot remove anymore apparently + # os.remove(pgpass_secret) + # shutil.move(pgpass_secret, home_folder) + + +def disable_base_modules(): + base_addons = os.environ.get('ODOO_BASE_PATH', '') + addons_to_remove = os.environ.get('ODOO_DISABLED_MODULES', '') + + modules = addons_to_remove.split(',') + modules = map(lambda mod: mod.strip(), modules) + + if not base_addons: + print("Do not attempt to remove wrong folder") + return + + for module in modules: + if not module: + continue + print("Removing module %s from %s" % (module, base_addons)) + + module_path = Path(base_addons, module) + if module_path.exists() and module_path.is_dir(): + shutil.rmtree(module_path) + else: + print("Module skipped as it doesn't seem to be present.") + + +def fix_access_rights(): + if os.environ.get('RESET_ACCESS_RIGHTS', '') == 'TRUE': + pipe(["chown", "-R", "odoo:odoo", "/var/lib/odoo"]) + pipe(["chown", "-R", "odoo:odoo", "/etc/odoo"]) + + +def remove_sudo(): + return pipe(["sed", "-i", "/odoo/d", "/etc/sudoers"]) + + +def main(): + install_apt_packages() + load_secrets() + fix_access_rights() + disable_base_modules() + return remove_sudo() + + +try: + code = main() + sys.exit(code) +except Exception as exc: + print(exc) + sys.exit(1) +except KeyboardInterrupt as exc: + print(exc) + sys.exit(1) diff --git a/8.0/Dockerfile b/8.0/Dockerfile index 4ee03445a..ba3d25cb9 100644 --- a/8.0/Dockerfile +++ b/8.0/Dockerfile @@ -3,18 +3,23 @@ LABEL maintainer="Archeti " # Generate locale C.UTF-8 for postgres and general locale data ENV LANG C.UTF-8 +ARG DEBIAN_FRONTEND=noninteractive +COPY ./platform.sh /platform.sh # Install some dependencies python3.7 RUN set -x; \ - apt-get update \ + export ARCH=`sh /platform.sh` \ + # Setup deb source for postgresql client + && apt-get update \ && apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg \ python-wheel \ python-setuptools \ python-pip \ python2.7 \ libpython2.7 \ curl \ - gnupg \ libpq-dev \ libsasl2-2 \ libldap-2.4-2 \ @@ -23,25 +28,23 @@ RUN set -x; \ libxslt1.1 \ sudo \ node-less \ - # python-yaml \ - && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb \ - && apt-get install -y --no-install-recommends ./wkhtmltox.deb \ + && echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ + && export GNUPGHOME="$(mktemp -d)" \ + && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ + && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ + && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ + && gpgconf --kill all \ + && rm -rf "$GNUPGHOME" \ + # Download deb for wkhtmltox + && echo "Downloading https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb" \ + && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + ./wkhtmltox.deb \ && rm -rf /var/lib/apt/lists/* wkhtmltox.deb -# Install latest postgresql-client -RUN set -x; \ - echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ - && export GNUPGHOME="$(mktemp -d)" \ - && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ - && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ - && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ - && gpgconf --kill all \ - && rm -rf "$GNUPGHOME" \ - && apt-get update \ - && apt-get install -y postgresql-client \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH=/usr/local/bin:$PATH +ENV PATH=/var/lib/odoo/.local/bin:/usr/local/bin:$PATH # Install Odoo Including things from sources ENV ODOO_VERSION 8.0 ENV ODOO_RELEASE=20171001 @@ -56,15 +59,14 @@ RUN set -x; \ libxml2-dev \ libxmlsec1-dev \ libxslt1-dev \ + git \ && pip install -U pip \ && /usr/bin/env pip install \ - psycogreen \ + odoo-tools==0.0.65 \ pathlib2 \ - && curl -o requirements.txt https://raw.githubusercontent.com/odoo/odoo/${ODOO_VERSION}/requirements.txt \ - && pip install -U pip \ - && pip install -r requirements.txt \ - && /usr/bin/env pip install https://nightly.odoo.com/${ODOO_VERSION}/nightly/src/${ODOO_ARCHIVE} \ - && cd / \ + && odoo-install --release "20171001" --version "8.0" --repo "https://github.com/odoo/odoo.git" --ref "" \ + && [ -f /usr/local/bin/odoo.py ] && ln -s /usr/local/bin/odoo.py /usr/local/bin/odoo \ + || cd / \ && apt-get --purge remove -y \ build-essential \ python2.7-dev \ @@ -73,17 +75,14 @@ RUN set -x; \ libxml2-dev \ libxmlsec1-dev \ libxslt1-dev \ + git \ && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* - -VOLUME /etc/odoo -VOLUME /var/lib/odoo + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /root/.cache COPY ./odoo.conf /etc/odoo/ COPY ./entrypoint.py / COPY ./sudo-entrypoint.py / -COPY ./find_modules.py /scripts/find_modules.py -COPY ./prepare_project.py /scripts/prepare_project.py ARG UID=1000 ARG GID=1000 @@ -93,10 +92,11 @@ RUN mkdir /addons \ && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ && chown odoo /etc/odoo/odoo.conf \ && chown -R odoo:odoo /addons \ - && cp /usr/local/bin/odoo.py /usr/local/bin/odoo || true \ && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /sudo-entrypoint.py" >> /etc/sudoers -ENV OPENERP_SERVER /etc/odoo/odoo.conf +VOLUME /etc/odoo +VOLUME /var/lib/odoo + ENV ODOO_RC /etc/odoo/odoo.conf ENV ODOO_BASE_PATH /usr/local/lib/python2.7/dist-packages/openerp/addons # Env variable defined to monitor the kind of service running @@ -113,13 +113,13 @@ USER odoo LABEL version="8.0" LABEL release="20171001" -LABEL org.opencontainers.image.created="2021-06-25T17:55:41.191604" +LABEL org.opencontainers.image.created="2022-11-04T20:06:13.797586" LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" -LABEL org.opencontainers.image.authors="Archeti " +LABEL org.opencontainers.image.authors="OdooPlus " LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.version="8.0" -LABEL org.opencontainers.image.vendor="ArcheTI" +LABEL org.opencontainers.image.vendor="OdooPlus" LABEL org.opencontainers.image.ref.name="8.0" LABEL org.opencontainers.image.title="Odoo 8.0" LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." diff --git a/8.0/entrypoint.py b/8.0/entrypoint.py index f2ce4654b..bb0ffddc3 100755 --- a/8.0/entrypoint.py +++ b/8.0/entrypoint.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # import argparse +from __future__ import print_function import time import shlex import subprocess @@ -17,25 +18,9 @@ from passlib.context import CryptContext from contextlib import contextmanager -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - -try: - from pip._internal.network.session import PipSession - from pip._internal.req.req_file import parse_requirements - from pip._internal.req.constructors import ( - install_req_from_parsed_requirement - ) -except Exception: - from pip.download import PipSession - from pip.req.req_file import parse_requirements - - def install_req_from_parsed_requirement(req): - return req - -from collections import defaultdict +from odoo_tools.compat import Path +from odoo_tools.modules.search import find_addons_paths, find_modules_paths +from odoo_tools.configuration.pips import merge_requirements try: # python3 @@ -68,63 +53,6 @@ def quote(s): return "'" + s.replace("'", "'\"'\"'") + "'" -class Requirement(object): - def __init__(self): - self.extras = set() - self.specifiers = set() - self.links = set() - self.editable = False - - -def merge_requirements(files): - requirements = defaultdict(lambda: Requirement()) - links = set() - - for filename in files: - f_requirements = parse_requirements( - str(filename), session=PipSession() - ) - for parsed_requirement in f_requirements: - requirement = install_req_from_parsed_requirement( - parsed_requirement - ) - - if requirement.markers and not requirement.markers.evaluate(): - continue - - if not hasattr(requirement.req, 'name'): - links.add(requirement.link.url) - break - name = requirement.req.name - specifiers = requirement.req.specifier - extras = requirement.req.extras - requirements[name].extras |= set(extras) - requirements[name].specifiers |= set(specifiers) - if requirement.link: - requirements[name].links |= {requirement.link.url} - requirements[name].editable |= requirement.editable - - result = [] - for key, value in requirements.items(): - if value.links: - result.append("%s" % value.links.pop()) - elif not value.extras: - result.append( - "%s %s" % (key, ",".join(map(str, value.specifiers))) - ) - else: - result.append("%s [%s] %s" % ( - key, - ",".join(map(str, value.extras)), - ",".join(map(str, value.specifiers)) - )) - - for link in links: - result.append(link) - - return "\n".join(result) - - def pipe(args): """ Call the process with std(in,out,err) @@ -178,42 +106,12 @@ def call_sudo_entrypoint(): return ret -def get_all_manifests(): - odoo_manifests = glob.glob('/addons/**/__manifest__.py', recursive=True) - erp_manifest = glob.glob('/addons/**/__openerp__.py', recursive=True) - - return odoo_manifests + erp_manifest - - -def module_name(manifest_file): - return path.basename(path.dirname(manifest_file)) - - -def get_module(manifest_file): - with open(manifest_file) as manifest_in: - try: - code = compile(manifest_in.read(), '__manifest__', 'eval') - module = eval(code, {}, {}) - except Exception: - module = {} - - return module - - -def is_installable(manifest): - return manifest.get('installable', True) - - -def is_server_wide(manifest): - return manifest.get('server_wide', False) - - def get_server_wide_modules(manifests): base_server_wide_modules = ['base', 'web'] custom_server_wide_modules = [ - module_name(filename) - for filename, module in manifests.items() - if is_server_wide(module) + manifest.technical_name + for manifest in manifests + if manifest.server_wide ] return base_server_wide_modules + custom_server_wide_modules @@ -246,7 +144,12 @@ def install_python_dependencies(valid_paths): req_file = Path('/var/lib/odoo/requirements.txt') with req_file.open('w') as fout: data = merge_requirements(requirement_files) - fout.write(data) + data = "\n".join(data) + + try: + fout.write(data) + except Exception: + fout.write(data.decode('utf-8')) for requirements in requirement_files: print("Installing python packages from %s" % requirements) @@ -338,65 +241,44 @@ def get_extra_paths(): ] -def get_valid_paths(): - base_addons = os.environ.get('ODOO_BASE_PATH') +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] - # addons = os.listdir('/addons') - valid_paths = [base_addons] +def get_valid_paths(): + base_addons = os.environ['ODOO_BASE_PATH'] - addons_paths = get_dirs('/addons') - addons_paths += get_extra_paths() + base_addons_paths = [base_addons, '/addons'] + base_addons_paths += get_extra_paths() - for addons_path in addons_paths: - print("Lookup addons in %s" % addons_path) - flush_streams() - addons = get_dirs(addons_path) - for addon in addons: - files = os.listdir(addon) - if ( - '__init__.py' in files and - '__manifest__.py' in files or - '__openerp__.py' in files - ): - valid_paths.append(addons_path) - print("Addons found !") - flush_streams() - break - if addons_path in valid_paths: - continue - else: - print("No addons found in path. Skipping...") - flush_streams() + valid_paths = find_addons_paths(base_addons_paths) + + for valid_path in valid_paths: + print("Found installable modules in {}".format(valid_path)) return valid_paths def setup_addons_paths(config, valid_paths): - config.set('options', 'addons_path', ",".join(valid_paths)) + config.set('options', 'addons_path', ",".join([ + str(path) for path in valid_paths + ])) flush_streams() -def setup_server_wide_modules(config): +def setup_server_wide_modules(config, valid_paths): print("Searching for server wide modules") - all_manifests = get_all_manifests() + filters = set(['installable']) + all_manifests = find_modules_paths(valid_paths, filters) - print("Found %s modules" % (len(all_manifests))) + print("Found %s installable modules " % (len(all_manifests),)) - manifests = { - filename: get_module(filename) - for filename in all_manifests - } - - installable_manifests = { - filename: module - for filename, module in manifests.items() - if is_installable(module) - } - - print("Found %s installable modules " % (len(installable_manifests),)) - - server_wide_modules = get_server_wide_modules(installable_manifests) + server_wide_modules = get_server_wide_modules(all_manifests) if len(server_wide_modules) > 2: modules = ",".join(server_wide_modules) @@ -408,6 +290,36 @@ def setup_server_wide_modules(config): flush_streams() +def convert_value(name, value): + if value in ['True', 'False']: + return value == 'True' + + return value + + +def setup_env_config(config): + try: + from odoo.tools import config as odoo_config + except ImportError: + from openerp.tools import config as odoo_config + + params_by_name = {} + for key, opt in odoo_config.casts.items(): + value_opt = opt.get_opt_string() + value_opt = value_opt.upper().replace('--', 'ODOO_') + value_opt = value_opt.replace('-', '_') + params_by_name[value_opt] = key + + for key, value in os.environ.items(): + if not key.startswith('ODOO_'): + continue + + if key in params_by_name: + config_name = params_by_name[key] + converted_value = convert_value(key, value) + config.set('options', config_name, converted_value) + + def setup_environ(config): print("Configuring environment variables for postgresql") @@ -499,12 +411,39 @@ def wait_postgresql(): @contextmanager def get_config(config_path): config = ConfigParser() - config.read(config_path) + config.read(str(config_path)) yield config - with config_path.open('w') as out: - config.write(out) + try: + with config_path.open('w') as out: + config.write(out) + except Exception: + with config_path.open('wb') as out: + config.write(out) + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for vpath in paths: + cur_path = Path(vpath) + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(vpath) + + return res_paths def main(): @@ -518,7 +457,9 @@ def main(): sys.exit(ret) # Install python packages with pip in user home - valid_paths = get_valid_paths() + orig_valid_paths = get_valid_paths() + excluded_paths = get_excluded_paths() + valid_paths = filter_valid_paths(orig_valid_paths, excluded_paths) config_path = Path(os.environ.get('ODOO_RC', '/etc/odoo/odoo.cfg')) @@ -528,7 +469,8 @@ def main(): install_master_password(config) setup_environ(config) setup_addons_paths(config, valid_paths) - setup_server_wide_modules(config) + setup_server_wide_modules(config, valid_paths) + setup_env_config(config) if not os.environ.get('ODOO_SKIP_POSTGRES_WAIT'): wait_postgresql() diff --git a/8.0/find_modules.py b/8.0/find_modules.py deleted file mode 100644 index 9cb943748..000000000 --- a/8.0/find_modules.py +++ /dev/null @@ -1,76 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -from itertools import chain - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - "-p", - dest='paths', - action="append", - help="Location in which to search", - default=[] - ) - - parser.add_option( - "--only-name", - dest="only_name", - action="store_true", - help="Only display module name instead of path", - default=False, - ) - - parser.add_option( - '--csv', - dest="is_csv", - action="store_true", - help="Output as a comma separated list", - default=False - ) - - return parser - - -def find_modules(options, path): - modules = set() - - path = Path.cwd() / path - - erp_manifest = '__openerp__.py' - odoo_manifest = '__manifest__.py' - - manifest_globs = chain( - path.glob('**/{}'.format(erp_manifest)), - path.glob('**/{}'.format(odoo_manifest)), - ) - - for path in manifest_globs: - rel_path = path.parent.relative_to(Path.cwd()) - if options.only_name: - modules.add(rel_path.name) - else: - modules.add(str(rel_path)) - - return modules - - -def main(options, args): - modules = set() - for path in options.paths: - modules = modules.union(find_modules(options, path)) - - return modules - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - modules = main(options, args) - - if not options.is_csv: - for module in modules: - print(module) - else: - print(",".join(modules), end="") diff --git a/8.0/platform.sh b/8.0/platform.sh new file mode 100644 index 000000000..fdf5e42eb --- /dev/null +++ b/8.0/platform.sh @@ -0,0 +1,13 @@ +PLATFORM=`uname -m` + +if [ $PLATFORM = "aarch64" ] +then + echo -n "arm64" +else +if [ $PLATFORM = "x86_64" ] +then + echo -n "amd64" +else + echo -n $PLATFORM +fi +fi diff --git a/8.0/prepare_project.py b/8.0/prepare_project.py deleted file mode 100644 index 74a53f2e0..000000000 --- a/8.0/prepare_project.py +++ /dev/null @@ -1,198 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -import toml -from giturlparse import parse -from contextlib import contextmanager -import os -from subprocess import run -from urllib.parse import urlparse - - -def get_services(services): - return { - service.get('name'): service - for service in services.get('services') - } - - -def addons_by_project(options, addons): - res = {} - - for addon in addons: - if addon['url'] == 'self': - if options.ignore_self: - continue - parsed = parse(options.url) - else: - parsed = parse(addon['url']) - - auth = parsed.protocol in ['git', 'ssh'] - - res[parsed.repo] = dict( - addon, - url=parsed.url2https, - auth=auth - ) - - return res - - -def merge_addons(options, base, other): - base_addons = addons_by_project(options, base) - other_addons = addons_by_project(options, other) - - for name, addon in other_addons.items(): - if name not in base_addons: - base_addons[name] = addon - else: - base_addons[name] = dict(base_addons[name], **addon) - - return [ - addon - for addon in base_addons.values() - ] - - -def merge_services(options, base, other): - basic_inherit = dict(base, **other) - - if base.get('addons') or other.get('addons'): - basic_inherit['addons'] = merge_addons( - options, - base.get('addons', []), - other.get('addons', []) - ) - - return basic_inherit - - -def compile_service(options, services, name): - service = services.get(name, {}) - if 'inherit' in service: - merge_service = compile_service(options, services, service['inherit']) - service = merge_services(options, merge_service, service) - - return service - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - '-f', - '--file', - dest="file", - help="Input File" - ) - - parser.add_option( - '--url', - dest='url', - help="Url of self project" - ) - - parser.add_option( - '-o', - '--output', - dest="output_directory", - help="Output Directory" - ) - - parser.add_option( - '-e', - dest="env", - help="Environment to prepare" - ) - - parser.add_option( - '--username', - dest="username", - help="Username to replace with", - ) - - parser.add_option( - '--password', - dest="password", - help="password to set on https urls" - ) - - parser.add_option( - '-b', - '--branch', - dest="branch", - help="Default branch if no ref is defined" - ) - - parser.add_option( - '--ignore-self', - dest="ignore_self", - action="store_true", - help="Ignore self url as it's already fetched", - default=False - ) - - return parser - - -@contextmanager -def cd(directory): - cwd = Path.cwd() - try: - os.chdir(directory) - yield - finally: - os.chdir(cwd) - - -def fetch_addons(options, addon): - parsed = parse(addon['url']) - - url = urlparse(parsed.url2https) - - if addon['auth'] and options.username and options.password: - url = url._replace( - netloc="{}:{}@{}".format( - options.username, - options.password, - url.netloc - ) - ) - - repo_path = Path.cwd() / options.output_directory / parsed.repo - - repo_path.mkdir(exist_ok=True) - - with cd(repo_path): - run(['git', 'init']) - run(['git', 'remote', 'add', 'origin', url.geturl()]) - - ref = addon.get('commit') or addon.get('branch') or options.branch - - if ref: - run(['git', 'fetch', 'origin', ref]) - else: - run(['git', 'fetch', 'origin']) - - run(['git', 'checkout', 'FETCH_HEAD']) - run(['git', 'remote', 'remove', 'origin']) - - -def main(options, args): - with Path(options.file).open('r') as fin: - services = toml.loads(fin.read()) - - by_name = get_services(services) - - outputdir = Path(options.output_directory) - outputdir.mkdir(exist_ok=True) - - service = compile_service(options, by_name, options.env) - - for addons in service.get('addons', []): - fetch_addons(options, addons) - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - main(options, args) diff --git a/8.0/sudo-entrypoint.py b/8.0/sudo-entrypoint.py index 98779baae..916239170 100755 --- a/8.0/sudo-entrypoint.py +++ b/8.0/sudo-entrypoint.py @@ -12,6 +12,7 @@ from os import path from os.path import expanduser import shutil +import six try: from pathlib import Path @@ -69,6 +70,37 @@ def get_addons_paths(): ] +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for cur_path in paths: + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(cur_path) + + return res_paths + + def install_apt_packages(): """ Install debian dependencies. @@ -76,26 +108,34 @@ def install_apt_packages(): package_list = set() paths = get_addons_paths() + excluded_paths = get_excluded_paths() + paths = filter_valid_paths(paths, excluded_paths) print("Looking up for packages in {}".format(paths)) for addons_path in paths: for packages in addons_path.glob('**/apt-packages.txt'): print("Installing packages from %s" % packages) - with open(packages, 'r') as pack_file: - lines = [line.strip() for line in pack_file] + with packages.open('r') as pack_file: + lines = [ + six.ensure_text(line).strip() + for line in pack_file + if six.ensure_text(line).strip() + ] package_list.update(set(lines)) extras = os.environ.get('EXTRA_APT_PACKAGES', '') - print(f"Adding extra packages {extras}") + print("Adding extra packages {extras}".format(extras=extras)) if extras: for package in extras.split(','): - if not package: + if not package.strip(): continue - package_list.add(package) + package_list.add( + six.ensure_text(package).strip() + ) if len(package_list) > 0: - print(f"Installing {package_list}") + print("Installing {package_list}".format(package_list=package_list)) ret = pipe(['apt-get', 'update']) # Something went wrong, stop the service as it's failing diff --git a/9.0/Dockerfile b/9.0/Dockerfile index e9042caa5..5f4cfb838 100644 --- a/9.0/Dockerfile +++ b/9.0/Dockerfile @@ -3,18 +3,23 @@ LABEL maintainer="Archeti " # Generate locale C.UTF-8 for postgres and general locale data ENV LANG C.UTF-8 +ARG DEBIAN_FRONTEND=noninteractive +COPY ./platform.sh /platform.sh # Install some dependencies python3.7 RUN set -x; \ - apt-get update \ + export ARCH=`sh /platform.sh` \ + # Setup deb source for postgresql client + && apt-get update \ && apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg \ python-wheel \ python-setuptools \ python-pip \ python2.7 \ libpython2.7 \ curl \ - gnupg \ libpq-dev \ libsasl2-2 \ libldap-2.4-2 \ @@ -23,25 +28,23 @@ RUN set -x; \ libxslt1.1 \ sudo \ node-less \ - # python-yaml \ - && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb \ - && apt-get install -y --no-install-recommends ./wkhtmltox.deb \ + && echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ + && export GNUPGHOME="$(mktemp -d)" \ + && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ + && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ + && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ + && gpgconf --kill all \ + && rm -rf "$GNUPGHOME" \ + # Download deb for wkhtmltox + && echo "Downloading https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb" \ + && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.bionic_${ARCH}.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + ./wkhtmltox.deb \ && rm -rf /var/lib/apt/lists/* wkhtmltox.deb -# Install latest postgresql-client -RUN set -x; \ - echo 'deb https://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > etc/apt/sources.list.d/pgdg.list \ - && export GNUPGHOME="$(mktemp -d)" \ - && repokey='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ - && gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "${repokey}" \ - && gpg --batch --armor --export "${repokey}" > /etc/apt/trusted.gpg.d/pgdg.gpg.asc \ - && gpgconf --kill all \ - && rm -rf "$GNUPGHOME" \ - && apt-get update \ - && apt-get install -y postgresql-client \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH=/usr/local/bin:$PATH +ENV PATH=/var/lib/odoo/.local/bin:/usr/local/bin:$PATH # Install Odoo Including things from sources ENV ODOO_VERSION 9.0c ENV ODOO_RELEASE=20190401 @@ -56,15 +59,14 @@ RUN set -x; \ libxml2-dev \ libxmlsec1-dev \ libxslt1-dev \ + git \ && pip install -U pip \ && /usr/bin/env pip install \ - psycogreen \ + odoo-tools==0.0.65 \ pathlib2 \ - && curl -o requirements.txt https://raw.githubusercontent.com/odoo/odoo/9.0/requirements.txt \ - && pip install -U pip \ - && pip install -r requirements.txt \ - && /usr/bin/env pip install https://nightly.odoo.com/9.0/nightly/src/${ODOO_ARCHIVE} \ - && cd / \ + && odoo-install --release "20190401" --version "9.0c" --repo "https://github.com/odoo/odoo.git" --ref "" \ + && [ -f /usr/local/bin/odoo.py ] && ln -s /usr/local/bin/odoo.py /usr/local/bin/odoo \ + || cd / \ && apt-get --purge remove -y \ build-essential \ python2.7-dev \ @@ -73,17 +75,14 @@ RUN set -x; \ libxml2-dev \ libxmlsec1-dev \ libxslt1-dev \ + git \ && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* - -VOLUME /etc/odoo -VOLUME /var/lib/odoo + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /root/.cache COPY ./odoo.conf /etc/odoo/ COPY ./entrypoint.py / COPY ./sudo-entrypoint.py / -COPY ./find_modules.py /scripts/find_modules.py -COPY ./prepare_project.py /scripts/prepare_project.py ARG UID=1000 ARG GID=1000 @@ -93,10 +92,11 @@ RUN mkdir /addons \ && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ && chown odoo /etc/odoo/odoo.conf \ && chown -R odoo:odoo /addons \ - && cp /usr/local/bin/odoo.py /usr/local/bin/odoo || true \ && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /sudo-entrypoint.py" >> /etc/sudoers -ENV OPENERP_SERVER /etc/odoo/odoo.conf +VOLUME /etc/odoo +VOLUME /var/lib/odoo + ENV ODOO_RC /etc/odoo/odoo.conf ENV ODOO_BASE_PATH /usr/local/lib/python2.7/dist-packages/openerp/addons # Env variable defined to monitor the kind of service running @@ -113,13 +113,13 @@ USER odoo LABEL version="9.0c" LABEL release="20190401" -LABEL org.opencontainers.image.created="2021-06-25T17:55:41.192956" +LABEL org.opencontainers.image.created="2022-11-04T20:06:13.798969" LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" -LABEL org.opencontainers.image.authors="Archeti " +LABEL org.opencontainers.image.authors="OdooPlus " LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.version="9.0c" -LABEL org.opencontainers.image.vendor="ArcheTI" +LABEL org.opencontainers.image.vendor="OdooPlus" LABEL org.opencontainers.image.ref.name="9.0" LABEL org.opencontainers.image.title="Odoo 9.0c" LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." diff --git a/9.0/entrypoint.py b/9.0/entrypoint.py index f2ce4654b..bb0ffddc3 100755 --- a/9.0/entrypoint.py +++ b/9.0/entrypoint.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # import argparse +from __future__ import print_function import time import shlex import subprocess @@ -17,25 +18,9 @@ from passlib.context import CryptContext from contextlib import contextmanager -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - -try: - from pip._internal.network.session import PipSession - from pip._internal.req.req_file import parse_requirements - from pip._internal.req.constructors import ( - install_req_from_parsed_requirement - ) -except Exception: - from pip.download import PipSession - from pip.req.req_file import parse_requirements - - def install_req_from_parsed_requirement(req): - return req - -from collections import defaultdict +from odoo_tools.compat import Path +from odoo_tools.modules.search import find_addons_paths, find_modules_paths +from odoo_tools.configuration.pips import merge_requirements try: # python3 @@ -68,63 +53,6 @@ def quote(s): return "'" + s.replace("'", "'\"'\"'") + "'" -class Requirement(object): - def __init__(self): - self.extras = set() - self.specifiers = set() - self.links = set() - self.editable = False - - -def merge_requirements(files): - requirements = defaultdict(lambda: Requirement()) - links = set() - - for filename in files: - f_requirements = parse_requirements( - str(filename), session=PipSession() - ) - for parsed_requirement in f_requirements: - requirement = install_req_from_parsed_requirement( - parsed_requirement - ) - - if requirement.markers and not requirement.markers.evaluate(): - continue - - if not hasattr(requirement.req, 'name'): - links.add(requirement.link.url) - break - name = requirement.req.name - specifiers = requirement.req.specifier - extras = requirement.req.extras - requirements[name].extras |= set(extras) - requirements[name].specifiers |= set(specifiers) - if requirement.link: - requirements[name].links |= {requirement.link.url} - requirements[name].editable |= requirement.editable - - result = [] - for key, value in requirements.items(): - if value.links: - result.append("%s" % value.links.pop()) - elif not value.extras: - result.append( - "%s %s" % (key, ",".join(map(str, value.specifiers))) - ) - else: - result.append("%s [%s] %s" % ( - key, - ",".join(map(str, value.extras)), - ",".join(map(str, value.specifiers)) - )) - - for link in links: - result.append(link) - - return "\n".join(result) - - def pipe(args): """ Call the process with std(in,out,err) @@ -178,42 +106,12 @@ def call_sudo_entrypoint(): return ret -def get_all_manifests(): - odoo_manifests = glob.glob('/addons/**/__manifest__.py', recursive=True) - erp_manifest = glob.glob('/addons/**/__openerp__.py', recursive=True) - - return odoo_manifests + erp_manifest - - -def module_name(manifest_file): - return path.basename(path.dirname(manifest_file)) - - -def get_module(manifest_file): - with open(manifest_file) as manifest_in: - try: - code = compile(manifest_in.read(), '__manifest__', 'eval') - module = eval(code, {}, {}) - except Exception: - module = {} - - return module - - -def is_installable(manifest): - return manifest.get('installable', True) - - -def is_server_wide(manifest): - return manifest.get('server_wide', False) - - def get_server_wide_modules(manifests): base_server_wide_modules = ['base', 'web'] custom_server_wide_modules = [ - module_name(filename) - for filename, module in manifests.items() - if is_server_wide(module) + manifest.technical_name + for manifest in manifests + if manifest.server_wide ] return base_server_wide_modules + custom_server_wide_modules @@ -246,7 +144,12 @@ def install_python_dependencies(valid_paths): req_file = Path('/var/lib/odoo/requirements.txt') with req_file.open('w') as fout: data = merge_requirements(requirement_files) - fout.write(data) + data = "\n".join(data) + + try: + fout.write(data) + except Exception: + fout.write(data.decode('utf-8')) for requirements in requirement_files: print("Installing python packages from %s" % requirements) @@ -338,65 +241,44 @@ def get_extra_paths(): ] -def get_valid_paths(): - base_addons = os.environ.get('ODOO_BASE_PATH') +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] - # addons = os.listdir('/addons') - valid_paths = [base_addons] +def get_valid_paths(): + base_addons = os.environ['ODOO_BASE_PATH'] - addons_paths = get_dirs('/addons') - addons_paths += get_extra_paths() + base_addons_paths = [base_addons, '/addons'] + base_addons_paths += get_extra_paths() - for addons_path in addons_paths: - print("Lookup addons in %s" % addons_path) - flush_streams() - addons = get_dirs(addons_path) - for addon in addons: - files = os.listdir(addon) - if ( - '__init__.py' in files and - '__manifest__.py' in files or - '__openerp__.py' in files - ): - valid_paths.append(addons_path) - print("Addons found !") - flush_streams() - break - if addons_path in valid_paths: - continue - else: - print("No addons found in path. Skipping...") - flush_streams() + valid_paths = find_addons_paths(base_addons_paths) + + for valid_path in valid_paths: + print("Found installable modules in {}".format(valid_path)) return valid_paths def setup_addons_paths(config, valid_paths): - config.set('options', 'addons_path', ",".join(valid_paths)) + config.set('options', 'addons_path', ",".join([ + str(path) for path in valid_paths + ])) flush_streams() -def setup_server_wide_modules(config): +def setup_server_wide_modules(config, valid_paths): print("Searching for server wide modules") - all_manifests = get_all_manifests() + filters = set(['installable']) + all_manifests = find_modules_paths(valid_paths, filters) - print("Found %s modules" % (len(all_manifests))) + print("Found %s installable modules " % (len(all_manifests),)) - manifests = { - filename: get_module(filename) - for filename in all_manifests - } - - installable_manifests = { - filename: module - for filename, module in manifests.items() - if is_installable(module) - } - - print("Found %s installable modules " % (len(installable_manifests),)) - - server_wide_modules = get_server_wide_modules(installable_manifests) + server_wide_modules = get_server_wide_modules(all_manifests) if len(server_wide_modules) > 2: modules = ",".join(server_wide_modules) @@ -408,6 +290,36 @@ def setup_server_wide_modules(config): flush_streams() +def convert_value(name, value): + if value in ['True', 'False']: + return value == 'True' + + return value + + +def setup_env_config(config): + try: + from odoo.tools import config as odoo_config + except ImportError: + from openerp.tools import config as odoo_config + + params_by_name = {} + for key, opt in odoo_config.casts.items(): + value_opt = opt.get_opt_string() + value_opt = value_opt.upper().replace('--', 'ODOO_') + value_opt = value_opt.replace('-', '_') + params_by_name[value_opt] = key + + for key, value in os.environ.items(): + if not key.startswith('ODOO_'): + continue + + if key in params_by_name: + config_name = params_by_name[key] + converted_value = convert_value(key, value) + config.set('options', config_name, converted_value) + + def setup_environ(config): print("Configuring environment variables for postgresql") @@ -499,12 +411,39 @@ def wait_postgresql(): @contextmanager def get_config(config_path): config = ConfigParser() - config.read(config_path) + config.read(str(config_path)) yield config - with config_path.open('w') as out: - config.write(out) + try: + with config_path.open('w') as out: + config.write(out) + except Exception: + with config_path.open('wb') as out: + config.write(out) + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for vpath in paths: + cur_path = Path(vpath) + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(vpath) + + return res_paths def main(): @@ -518,7 +457,9 @@ def main(): sys.exit(ret) # Install python packages with pip in user home - valid_paths = get_valid_paths() + orig_valid_paths = get_valid_paths() + excluded_paths = get_excluded_paths() + valid_paths = filter_valid_paths(orig_valid_paths, excluded_paths) config_path = Path(os.environ.get('ODOO_RC', '/etc/odoo/odoo.cfg')) @@ -528,7 +469,8 @@ def main(): install_master_password(config) setup_environ(config) setup_addons_paths(config, valid_paths) - setup_server_wide_modules(config) + setup_server_wide_modules(config, valid_paths) + setup_env_config(config) if not os.environ.get('ODOO_SKIP_POSTGRES_WAIT'): wait_postgresql() diff --git a/9.0/find_modules.py b/9.0/find_modules.py deleted file mode 100644 index 9cb943748..000000000 --- a/9.0/find_modules.py +++ /dev/null @@ -1,76 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -from itertools import chain - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - "-p", - dest='paths', - action="append", - help="Location in which to search", - default=[] - ) - - parser.add_option( - "--only-name", - dest="only_name", - action="store_true", - help="Only display module name instead of path", - default=False, - ) - - parser.add_option( - '--csv', - dest="is_csv", - action="store_true", - help="Output as a comma separated list", - default=False - ) - - return parser - - -def find_modules(options, path): - modules = set() - - path = Path.cwd() / path - - erp_manifest = '__openerp__.py' - odoo_manifest = '__manifest__.py' - - manifest_globs = chain( - path.glob('**/{}'.format(erp_manifest)), - path.glob('**/{}'.format(odoo_manifest)), - ) - - for path in manifest_globs: - rel_path = path.parent.relative_to(Path.cwd()) - if options.only_name: - modules.add(rel_path.name) - else: - modules.add(str(rel_path)) - - return modules - - -def main(options, args): - modules = set() - for path in options.paths: - modules = modules.union(find_modules(options, path)) - - return modules - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - modules = main(options, args) - - if not options.is_csv: - for module in modules: - print(module) - else: - print(",".join(modules), end="") diff --git a/9.0/platform.sh b/9.0/platform.sh new file mode 100644 index 000000000..fdf5e42eb --- /dev/null +++ b/9.0/platform.sh @@ -0,0 +1,13 @@ +PLATFORM=`uname -m` + +if [ $PLATFORM = "aarch64" ] +then + echo -n "arm64" +else +if [ $PLATFORM = "x86_64" ] +then + echo -n "amd64" +else + echo -n $PLATFORM +fi +fi diff --git a/9.0/prepare_project.py b/9.0/prepare_project.py deleted file mode 100644 index 74a53f2e0..000000000 --- a/9.0/prepare_project.py +++ /dev/null @@ -1,198 +0,0 @@ -from optparse import OptionParser -from pathlib import Path -import toml -from giturlparse import parse -from contextlib import contextmanager -import os -from subprocess import run -from urllib.parse import urlparse - - -def get_services(services): - return { - service.get('name'): service - for service in services.get('services') - } - - -def addons_by_project(options, addons): - res = {} - - for addon in addons: - if addon['url'] == 'self': - if options.ignore_self: - continue - parsed = parse(options.url) - else: - parsed = parse(addon['url']) - - auth = parsed.protocol in ['git', 'ssh'] - - res[parsed.repo] = dict( - addon, - url=parsed.url2https, - auth=auth - ) - - return res - - -def merge_addons(options, base, other): - base_addons = addons_by_project(options, base) - other_addons = addons_by_project(options, other) - - for name, addon in other_addons.items(): - if name not in base_addons: - base_addons[name] = addon - else: - base_addons[name] = dict(base_addons[name], **addon) - - return [ - addon - for addon in base_addons.values() - ] - - -def merge_services(options, base, other): - basic_inherit = dict(base, **other) - - if base.get('addons') or other.get('addons'): - basic_inherit['addons'] = merge_addons( - options, - base.get('addons', []), - other.get('addons', []) - ) - - return basic_inherit - - -def compile_service(options, services, name): - service = services.get(name, {}) - if 'inherit' in service: - merge_service = compile_service(options, services, service['inherit']) - service = merge_services(options, merge_service, service) - - return service - - -def get_parser(): - parser = OptionParser() - - parser.add_option( - '-f', - '--file', - dest="file", - help="Input File" - ) - - parser.add_option( - '--url', - dest='url', - help="Url of self project" - ) - - parser.add_option( - '-o', - '--output', - dest="output_directory", - help="Output Directory" - ) - - parser.add_option( - '-e', - dest="env", - help="Environment to prepare" - ) - - parser.add_option( - '--username', - dest="username", - help="Username to replace with", - ) - - parser.add_option( - '--password', - dest="password", - help="password to set on https urls" - ) - - parser.add_option( - '-b', - '--branch', - dest="branch", - help="Default branch if no ref is defined" - ) - - parser.add_option( - '--ignore-self', - dest="ignore_self", - action="store_true", - help="Ignore self url as it's already fetched", - default=False - ) - - return parser - - -@contextmanager -def cd(directory): - cwd = Path.cwd() - try: - os.chdir(directory) - yield - finally: - os.chdir(cwd) - - -def fetch_addons(options, addon): - parsed = parse(addon['url']) - - url = urlparse(parsed.url2https) - - if addon['auth'] and options.username and options.password: - url = url._replace( - netloc="{}:{}@{}".format( - options.username, - options.password, - url.netloc - ) - ) - - repo_path = Path.cwd() / options.output_directory / parsed.repo - - repo_path.mkdir(exist_ok=True) - - with cd(repo_path): - run(['git', 'init']) - run(['git', 'remote', 'add', 'origin', url.geturl()]) - - ref = addon.get('commit') or addon.get('branch') or options.branch - - if ref: - run(['git', 'fetch', 'origin', ref]) - else: - run(['git', 'fetch', 'origin']) - - run(['git', 'checkout', 'FETCH_HEAD']) - run(['git', 'remote', 'remove', 'origin']) - - -def main(options, args): - with Path(options.file).open('r') as fin: - services = toml.loads(fin.read()) - - by_name = get_services(services) - - outputdir = Path(options.output_directory) - outputdir.mkdir(exist_ok=True) - - service = compile_service(options, by_name, options.env) - - for addons in service.get('addons', []): - fetch_addons(options, addons) - - -if __name__ == '__main__': - parser = get_parser() - (options, args) = parser.parse_args() - main(options, args) diff --git a/9.0/sudo-entrypoint.py b/9.0/sudo-entrypoint.py index 98779baae..916239170 100755 --- a/9.0/sudo-entrypoint.py +++ b/9.0/sudo-entrypoint.py @@ -12,6 +12,7 @@ from os import path from os.path import expanduser import shutil +import six try: from pathlib import Path @@ -69,6 +70,37 @@ def get_addons_paths(): ] +def get_excluded_paths(): + excluded_paths = os.environ.get('ODOO_EXCLUDED_PATHS', '') + return [ + Path(x.strip()) + for x in excluded_paths.split(',') + if x + ] + + +def is_subdir_of(path1, path2): + try: + path2.relative_to(path1) + except ValueError: + return False + else: + return True + + +def filter_valid_paths(paths, excluded_paths): + res_paths = [] + + for cur_path in paths: + for ex_path in excluded_paths: + if is_subdir_of(ex_path, cur_path): + break + else: + res_paths.append(cur_path) + + return res_paths + + def install_apt_packages(): """ Install debian dependencies. @@ -76,26 +108,34 @@ def install_apt_packages(): package_list = set() paths = get_addons_paths() + excluded_paths = get_excluded_paths() + paths = filter_valid_paths(paths, excluded_paths) print("Looking up for packages in {}".format(paths)) for addons_path in paths: for packages in addons_path.glob('**/apt-packages.txt'): print("Installing packages from %s" % packages) - with open(packages, 'r') as pack_file: - lines = [line.strip() for line in pack_file] + with packages.open('r') as pack_file: + lines = [ + six.ensure_text(line).strip() + for line in pack_file + if six.ensure_text(line).strip() + ] package_list.update(set(lines)) extras = os.environ.get('EXTRA_APT_PACKAGES', '') - print(f"Adding extra packages {extras}") + print("Adding extra packages {extras}".format(extras=extras)) if extras: for package in extras.split(','): - if not package: + if not package.strip(): continue - package_list.add(package) + package_list.add( + six.ensure_text(package).strip() + ) if len(package_list) > 0: - print(f"Installing {package_list}") + print("Installing {package_list}".format(package_list=package_list)) ret = pipe(['apt-get', 'update']) # Something went wrong, stop the service as it's failing diff --git a/deploy.py b/deploy.py old mode 100644 new mode 100755 diff --git a/templates/Dockerfile.template b/templates/Dockerfile.template index 139633562..28c4eff46 100644 --- a/templates/Dockerfile.template +++ b/templates/Dockerfile.template @@ -18,13 +18,21 @@ RUN set -x; \ python3-wheel \ python3-setuptools \ python3-pip \ + python3-psycopg2 \ + python3-ldap \ + python3-libsass \ + python3-lxml \ + python3-pillow \ + python3-pypdf2 \ + python3-psutil \ + python3-asn1crypto \ curl \ - libpq-dev \ - libsasl2-2 \ - libldap-2.4-2 \ - libxml2 \ - libxmlsec1 \ - libxslt1.1 \ + # libpq-dev \ + # libsasl2-2 \ + # libldap-2.4-2 \ + # libxml2 \ + # libxmlsec1 \ + # libxslt1.1 \ sudo \ node-less \ && echo 'deb https://apt.postgresql.org/pub/repos/apt/ %(os_release)s-pgdg main' > etc/apt/sources.list.d/pgdg.list \ @@ -54,36 +62,36 @@ ARG ODOO_ARCHIVE=%(filename)s.tar.gz RUN set -x; \ apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential \ - %(python_version)s-dev \ - libsasl2-dev \ - libldap2-dev %(apt_packages)s \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ + # build-essential \ + # %(python_version)s-dev \ + # libsasl2-dev \ + # libldap2-dev %(apt_packages)s \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ git \ && pip install -U pip \ && /usr/bin/env pip install \ odoo-tools \ %(pip_packages)s \ - && odoo-install --release "%(release)s" --version "%(version)s" --repo "%(odoo_repo)s" --ref "%(ref)s" \ + && odootools manage setup --release "%(release)s" --repo "%(odoo_repo)s" --ref "%(ref)s" "%(version)s" \ && cd / \ && apt-get --purge remove -y \ - build-essential \ - %(python_version)s-dev \ - libsasl2-dev \ - libldap2-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev \ + # build-essential \ + # %(python_version)s-dev \ + # libsasl2-dev \ + # libldap2-dev \ + # libxml2-dev \ + # libxmlsec1-dev \ + # libxslt1-dev \ git \ && apt-get autoremove -y \ && rm -rf /var/lib/apt/lists/* \ && rm -rf /root/.cache -COPY ./%(config)s /etc/odoo/ -COPY ./%(entrypoint)s / -COPY ./sudo-%(entrypoint)s / +# COPY ./%(config)s /etc/odoo/ +# COPY ./%(entrypoint)s / +# COPY ./sudo-%(entrypoint)s / ARG UID=%(uid)d ARG GID=%(gid)d @@ -91,14 +99,14 @@ ARG GID=%(gid)d RUN mkdir /addons \ && groupadd -r -g ${GID} odoo \ && useradd -r -u ${UID} -g odoo -b /var/lib -m odoo \ - && chown odoo /etc/odoo/odoo.conf \ + # && chown odoo /etc/odoo/odoo.conf \ && chown -R odoo:odoo /addons \ - && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /sudo-entrypoint.py" >> /etc/sudoers + && echo "odoo ALL=(ALL:ALL) NOPASSWD:SETENV: /usr/local/bin/odootools" >> /etc/sudoers -VOLUME /etc/odoo +# VOLUME /etc/odoo VOLUME /var/lib/odoo -ENV ODOO_RC /etc/odoo/odoo.conf +ENV ODOO_RC /var/lib/odoo/odoo.conf ENV ODOO_BASE_PATH /usr/local/lib/%(python_version)s/dist-packages/odoo/addons # Env variable defined to monitor the kind of service running # it could be a staging/production/test or anything and undefined @@ -116,15 +124,15 @@ LABEL release="%(release)s" LABEL org.opencontainers.image.created="%(created_date)s" LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" -LABEL org.opencontainers.image.authors="Archeti " +LABEL org.opencontainers.image.authors="OdooPlus " LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.version="%(version)s" -LABEL org.opencontainers.image.vendor="ArcheTI" +LABEL org.opencontainers.image.vendor="OdooPlus" LABEL org.opencontainers.image.ref.name="%(tag)s" LABEL org.opencontainers.image.title="Odoo %(version)s" LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." -ENTRYPOINT ["/entrypoint.py"] +ENTRYPOINT ["odootools", "--log-level", "INFO", "entrypoint", "user"] cmd ["odoo"] diff --git a/templates/Dockerfile27.template b/templates/Dockerfile27.template index 8252acc86..d59f33f82 100644 --- a/templates/Dockerfile27.template +++ b/templates/Dockerfile27.template @@ -62,7 +62,7 @@ RUN set -x; \ git \ && pip install -U pip \ && /usr/bin/env pip install \ - odoo-tools \ + odoo-tools==0.0.65 \ %(pip_packages)s \ && odoo-install --release "%(release)s" --version "%(version)s" --repo "%(odoo_repo)s" --ref "%(ref)s" \ && [ -f /usr/local/bin/odoo.py ] && ln -s /usr/local/bin/odoo.py /usr/local/bin/odoo \ @@ -115,11 +115,11 @@ LABEL release="%(release)s" LABEL org.opencontainers.image.created="%(created_date)s" LABEL org.opencontainers.image.url="https://hub.docker.com/r/llacroix/odoo" -LABEL org.opencontainers.image.authors="Archeti " +LABEL org.opencontainers.image.authors="OdooPlus " LABEL org.opencontainers.image.documentation="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.source="https://github.com/llacroix/odoo-docker" LABEL org.opencontainers.image.version="%(version)s" -LABEL org.opencontainers.image.vendor="ArcheTI" +LABEL org.opencontainers.image.vendor="OdooPlus" LABEL org.opencontainers.image.ref.name="%(tag)s" LABEL org.opencontainers.image.title="Odoo %(version)s" LABEL org.opencontainers.image.description="Full featured odoo image that make odoo deployment fun and secure." diff --git a/versions.toml b/versions.toml index 69e7b41a6..b1cfe2f7f 100644 --- a/versions.toml +++ b/versions.toml @@ -11,7 +11,7 @@ module_name = "odoo" base_image = "ubuntu:bionic" os_release = "bionic" - python_version = "python3.6" + python_version = "python3.7" release = "" wkhtmltox_repo = "https://github.com/wkhtmltopdf/wkhtmltopdf" @@ -44,27 +44,46 @@ [odoo."10.0"] version = "10.0" - release = "20200313" + release = "20201204" template = "Dockerfile27.template" apt_packages = "ruby-sass" pip_packages = "pathlib2" + python_version = "python3.6" [odoo."11.0"] version = "11.0" - release = "20201204" + release = "20210922" apt_packages = "ruby-sass" + python_version = "python3.6" [odoo."12.0"] version = "12.0" - release = "20200623" + release = "20211011" [odoo."13.0"] version = "13.0" - release = "20200623" + release = "20221104" [odoo."14.0"] version = "14.0" - release = "20210212" + release = "20221104" + base_image = "ubuntu:focal" + os_release = "focal" + python_version = "python3.8" + +[odoo."15.0"] + version = "15.0" + release = "20221104" + base_image = "ubuntu:focal" + os_release = "focal" + python_version = "python3.8" + +[odoo."16.0"] + version = "16.0" + release = "20221104" + base_image = "ubuntu:focal" + os_release = "focal" + python_version = "python3.8" #filename = "odoo_14.0alpha1.${ODOO_RELEASE}" # version_path = "master" @@ -89,21 +108,14 @@ os_release = "focal" python_version = "python3.8" -[odoo."14.0-tiny"] - version = "14.0" - release = "20201009" - #filename = "odoo_14.0alpha1.${ODOO_RELEASE}" - template = "DockerfileTiny.template" - #version_path = "master" - -[odoo."14.3"] - version = "14.3" - release = "20210402" - filename = "odoo_14.3alpha1.${ODOO_RELEASE}" - version_path = "master" - [odoo."15.0-nightly"] version = "15.0" base_image = "ubuntu:focal" os_release = "focal" python_version = "python3.8" + +[odoo."16.0-nightly"] + version = "16.0" + base_image = "ubuntu:focal" + os_release = "focal" + python_version = "python3.8"