From b4bfb025f7ab0878e8e7e980dbad5b0a5bed1555 Mon Sep 17 00:00:00 2001

From: Lennart Poettering <lennart@poettering.net>

Date: Thu, 10 Dec 2015 12:40:04 +0100

Subject: [PATCH] importd: drop dkr support

The current code is not compatible with current dkr protocols anyway,

and dkr has a different focus ("microservices") than nspawn anyway

("whole machine containers"), hence drop support for it, we cannot

reasonably keep this up to date, and it creates the impression we'd

actually care for the microservices usecase.

Cherry-picked from: b43d75c

Related: #1284974

---

 Makefile.am                             |   2 -

 TODO                                    |   4 -

 configure.ac                            |   9 -

 man/machinectl.xml                      |  58 ---

 src/import/import-dkr.c                 | 891 --------------------------------

 src/import/import-dkr.h                 |  36 --

 src/import/importd.c                    | 111 +---

 src/import/org.freedesktop.import1.conf |   4 -

 src/import/pull.c                       | 119 +----

 src/machine/machinectl.c                |  90 +---

 src/shared/import-util.c                |  31 --

 src/shared/import-util.h                |   4 -

 12 files changed, 5 insertions(+), 1354 deletions(-)

 delete mode 100644 src/import/import-dkr.c

 delete mode 100644 src/import/import-dkr.h

diff --git a/Makefile.am b/Makefile.am

index 3a09e0a62..b0a34b212 100644

--- a/Makefile.am

+++ b/Makefile.am

@@ -5369,8 +5369,6 @@ systemd_pull_SOURCES = \

 	src/import/import-raw.h \

 	src/import/import-tar.c \

 	src/import/import-tar.h \

-	src/import/import-dkr.c \

-	src/import/import-dkr.h \

 	src/import/import-job.c \

 	src/import/import-job.h \

 	src/import/import-common.c \

diff --git a/TODO b/TODO

index 90b2c4b30..d96d2bf0e 100644

--- a/TODO

+++ b/TODO

@@ -126,10 +126,6 @@ Features:

 * rework journald sigbus stuff to use mutex

-* import-dkr: support tarsum checksum verification, if it becomes reality one day...

-

-* import-dkr: convert json bits to nspawn configuration

-

 * import: support import from local files, and export to local files

 * core/cgroup: support net_cls modules, and support automatically allocating class ids, then add support for making firewall changes depending on it, to implement a per-service firewall

diff --git a/configure.ac b/configure.ac

index 9103f9b92..2734368dc 100644

--- a/configure.ac

+++ b/configure.ac

@@ -1389,14 +1389,6 @@ AC_ARG_ENABLE([split-usr],

                 enable_split_usr=no

         ])])

-AC_ARG_WITH([dkr-index-url],

-        [AS_HELP_STRING([--dkr-index-url=URL], [Specify the default index URL to use for image downloads])],

-        [DEFAULT_DKR_INDEX_URL="\"$withval\""],

-        [DEFAULT_DKR_INDEX_URL="NULL"])

-

-AC_DEFINE_UNQUOTED(DEFAULT_DKR_INDEX_URL, [$DEFAULT_DKR_INDEX_URL], [Default index URL to use for image downloads])

-AC_SUBST(DEFAULT_DKR_INDEX_URL)

-

 AS_IF([test "x${enable_split_usr}" = "xyes"], [

         AC_DEFINE(HAVE_SPLIT_USR, 1, [Define if /bin, /sbin aren't symlinks into /usr])

 ])

@@ -1564,7 +1556,6 @@ AC_MSG_RESULT([

         Maximum System UID:      ${SYSTEM_UID_MAX}

         Maximum System GID:      ${SYSTEM_GID_MAX}

         Certificate root:        ${CERTIFICATEROOT}

-        Default dkr Index        ${DEFAULT_DKR_INDEX_URL}

         CFLAGS:                  ${OUR_CFLAGS} ${CFLAGS}

         CPPFLAGS:                ${OUR_CPPFLAGS} ${CPPFLAGS}

diff --git a/man/machinectl.xml b/man/machinectl.xml

index 640cb8b7d..b0a7f2ae4 100644

--- a/man/machinectl.xml

+++ b/man/machinectl.xml

@@ -204,16 +204,6 @@

         image.</para></listitem>

       </varlistentry>

-      <varlistentry>

-        <term><option>--dkr-index-url</option></term>

-

-        <listitem><para>Specifies the index server to use for

-        downloading <literal>dkr</literal> images with the

-        <command>pull-dkr</command>. Takes a

-        <literal>http://</literal>, <literal>https://</literal>

-        URL.</para></listitem>

-      </varlistentry>

-

       <xi:include href="user-system-options.xml" xpointer="host" />

       <xi:include href="user-system-options.xml" xpointer="machine" />

@@ -602,42 +592,6 @@

         below.</para></listitem>

       </varlistentry>

-      <varlistentry>

-        <term><command>pull-dkr</command> <replaceable>REMOTE</replaceable> [<replaceable>NAME</replaceable>]</term>

-

-        <listitem><para>Downloads a <literal>dkr</literal> container

-        image and makes it available locally. The remote name refers

-        to a <literal>dkr</literal> container name. If omitted, the

-        local machine name is derived from the <literal>dkr</literal>

-        container name.</para>

-

-        <para>Image verification is not available for

-        <literal>dkr</literal> containers, and thus

-        <option>--verify=no</option> must always be specified with

-        this command.</para>

-

-        <para>This command downloads all (missing) layers for the

-        specified container and places them in read-only subvolumes in

-        <filename>/var/lib/machines/</filename>. A writable snapshot

-        of the newest layer is then created under the specified local

-        machine name. To omit creation of this writable snapshot, pass

-        <literal>-</literal> as local machine name.</para>

-

-        <para>The read-only layer subvolumes are prefixed with

-        <filename>.dkr-</filename>, and thus now shown by

-        <command>list-images</command>, unless <option>--all</option>

-        is passed.</para>

-

-        <para>To specify the <literal>dkr</literal> index server to

-        use for looking up the specified container, use

-        <option>--dkr-index-url=</option>.</para>

-

-        <para>Note that pressing C-c during execution of this command

-        will not abort the download. Use

-        <command>cancel-transfer</command>, described

-        below.</para></listitem>

-      </varlistentry>

-

       <varlistentry>

         <term><command>list-transfers</command></term>

@@ -728,18 +682,6 @@

       the machine started as system service. With the last command a

       login prompt into the container is requested.</para>

     </example>

-

-    <example>

-      <title>Download a Fedora <literal>dkr</literal> image</title>

-

-      <programlisting># machinectl pull-dkr --verify=no mattdm/fedora

-# systemd-nspawn -M fedora</programlisting>

-

-      <para>Downloads a <literal>dkr</literal> image and opens a shell

-      in it. Note that the specified download command might require an

-      index server to be specified with the

-      <literal>--dkr-index-url=</literal>.</para>

-    </example>

   </refsect1>

   <refsect1>

diff --git a/src/import/import-dkr.c b/src/import/import-dkr.c

deleted file mode 100644

index fb72f6cee..000000000

--- a/src/import/import-dkr.c

+++ /dev/null

@@ -1,891 +0,0 @@

-/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/

-

-/***

-  This file is part of systemd.

-

-  Copyright 2014 Lennart Poettering

-

-  systemd is free software; you can redistribute it and/or modify it

-  under the terms of the GNU Lesser General Public License as published by

-  the Free Software Foundation; either version 2.1 of the License, or

-  (at your option) any later version.

-

-  systemd is distributed in the hope that it will be useful, but

-  WITHOUT ANY WARRANTY; without even the implied warranty of

-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU

-  Lesser General Public License for more details.

-

-  You should have received a copy of the GNU Lesser General Public License

-  along with systemd; If not, see <http://www.gnu.org/licenses/>.

-***/

-

-#include <curl/curl.h>

-#include <sys/prctl.h>

-

-#include "sd-daemon.h"

-#include "json.h"

-#include "strv.h"

-#include "btrfs-util.h"

-#include "utf8.h"

-#include "mkdir.h"

-#include "import-util.h"

-#include "curl-util.h"

-#include "aufs-util.h"

-#include "import-job.h"

-#include "import-common.h"

-#include "import-dkr.h"

-

-typedef enum DkrProgress {

-        DKR_SEARCHING,

-        DKR_RESOLVING,

-        DKR_METADATA,

-        DKR_DOWNLOADING,

-        DKR_COPYING,

-} DkrProgress;

-

-struct DkrImport {

-        sd_event *event;

-        CurlGlue *glue;

-

-        char *index_url;

-        char *image_root;

-

-        ImportJob *images_job;

-        ImportJob *tags_job;

-        ImportJob *ancestry_job;

-        ImportJob *json_job;

-        ImportJob *layer_job;

-

-        char *name;

-        char *tag;

-        char *id;

-

-        char *response_token;

-        char **response_registries;

-

-        char **ancestry;

-        unsigned n_ancestry;

-        unsigned current_ancestry;

-

-        DkrImportFinished on_finished;

-        void *userdata;

-

-        char *local;

-        bool force_local;

-

-        char *temp_path;

-        char *final_path;

-

-        pid_t tar_pid;

-};

-

-#define PROTOCOL_PREFIX "https://"

-

-#define HEADER_TOKEN "X-Do" /* the HTTP header for the auth token */ "cker-Token:"

-#define HEADER_REGISTRY "X-Do" /*the HTTP header for the registry */ "cker-Endpoints:"

-

-#define LAYERS_MAX 2048

-

-static void dkr_import_job_on_finished(ImportJob *j);

-

-DkrImport* dkr_import_unref(DkrImport *i) {

-        if (!i)

-                return NULL;

-

-        if (i->tar_pid > 1) {

-                (void) kill_and_sigcont(i->tar_pid, SIGKILL);

-                (void) wait_for_terminate(i->tar_pid, NULL);

-        }

-

-        import_job_unref(i->images_job);

-        import_job_unref(i->tags_job);

-        import_job_unref(i->ancestry_job);

-        import_job_unref(i->json_job);

-        import_job_unref(i->layer_job);

-

-        curl_glue_unref(i->glue);

-        sd_event_unref(i->event);

-

-        if (i->temp_path) {

-                (void) btrfs_subvol_remove(i->temp_path);

-                (void) rm_rf_dangerous(i->temp_path, false, true, false);

-                free(i->temp_path);

-        }

-

-        free(i->name);

-        free(i->tag);

-        free(i->id);

-        free(i->response_token);

-        free(i->response_registries);

-        strv_free(i->ancestry);

-        free(i->final_path);

-        free(i->index_url);

-        free(i->image_root);

-        free(i->local);

-        free(i);

-

-        return NULL;

-}

-

-int dkr_import_new(

-                DkrImport **ret,

-                sd_event *event,

-                const char *index_url,

-                const char *image_root,

-                DkrImportFinished on_finished,

-                void *userdata) {

-

-        _cleanup_(dkr_import_unrefp) DkrImport *i = NULL;

-        char *e;

-        int r;

-

-        assert(ret);

-        assert(index_url);

-

-        if (!http_url_is_valid(index_url))

-                return -EINVAL;

-

-        i = new0(DkrImport, 1);

-        if (!i)

-                return -ENOMEM;

-

-        i->on_finished = on_finished;

-        i->userdata = userdata;

-

-        i->image_root = strdup(image_root ?: "/var/lib/machines");

-        if (!i->image_root)

-                return -ENOMEM;

-

-        i->index_url = strdup(index_url);

-        if (!i->index_url)

-                return -ENOMEM;

-

-        e = endswith(i->index_url, "/");

-        if (e)

-                *e = 0;

-

-        if (event)

-                i->event = sd_event_ref(event);

-        else {

-                r = sd_event_default(&i->event);

-                if (r < 0)

-                        return r;

-        }

-

-        r = curl_glue_new(&i->glue, i->event);

-        if (r < 0)

-                return r;

-

-        i->glue->on_finished = import_job_curl_on_finished;

-        i->glue->userdata = i;

-

-        *ret = i;

-        i = NULL;

-

-        return 0;

-}

-

-static void dkr_import_report_progress(DkrImport *i, DkrProgress p) {

-        unsigned percent;

-

-        assert(i);

-

-        switch (p) {

-

-        case DKR_SEARCHING:

-                percent = 0;

-                if (i->images_job)

-                        percent += i->images_job->progress_percent * 5 / 100;

-                break;

-

-        case DKR_RESOLVING:

-                percent = 5;

-                if (i->tags_job)

-                        percent += i->tags_job->progress_percent * 5 / 100;

-                break;

-

-        case DKR_METADATA:

-                percent = 10;

-                if (i->ancestry_job)

-                        percent += i->ancestry_job->progress_percent * 5 / 100;

-                if (i->json_job)

-                        percent += i->json_job->progress_percent * 5 / 100;

-                break;

-

-        case DKR_DOWNLOADING:

-                percent = 20;

-                percent += 75 * i->current_ancestry / MAX(1U, i->n_ancestry);

-                if (i->layer_job)

-                        percent += i->layer_job->progress_percent * 75 / MAX(1U, i->n_ancestry) / 100;

-

-                break;

-

-        case DKR_COPYING:

-                percent = 95;

-                break;

-

-        default:

-                assert_not_reached("Unknown progress state");

-        }

-

-        sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);

-        log_debug("Combined progress %u%%", percent);

-}

-

-static int parse_id(const void *payload, size_t size, char **ret) {

-        _cleanup_free_ char *buf = NULL, *id = NULL, *other = NULL;

-        union json_value v = {};

-        void *json_state = NULL;

-        const char *p;

-        int t;

-

-        assert(payload);

-        assert(ret);

-

-        if (size <= 0)

-                return -EBADMSG;

-

-        if (memchr(payload, 0, size))

-                return -EBADMSG;

-

-        buf = strndup(payload, size);

-        if (!buf)

-                return -ENOMEM;

-

-        p = buf;

-        t = json_tokenize(&p, &id, &v, &json_state, NULL);

-        if (t < 0)

-                return t;

-        if (t != JSON_STRING)

-                return -EBADMSG;

-

-        t = json_tokenize(&p, &other, &v, &json_state, NULL);

-        if (t < 0)

-                return t;

-        if (t != JSON_END)

-                return -EBADMSG;

-

-        if (!dkr_id_is_valid(id))

-                return -EBADMSG;

-

-        *ret = id;

-        id = NULL;

-

-        return 0;

-}

-

-static int parse_ancestry(const void *payload, size_t size, char ***ret) {

-        _cleanup_free_ char *buf = NULL;

-        void *json_state = NULL;

-        const char *p;

-        enum {

-                STATE_BEGIN,

-                STATE_ITEM,

-                STATE_COMMA,

-                STATE_END,

-        } state = STATE_BEGIN;

-        _cleanup_strv_free_ char **l = NULL;

-        size_t n = 0, allocated = 0;

-

-        if (size <= 0)

-                return -EBADMSG;

-

-        if (memchr(payload, 0, size))

-                return -EBADMSG;

-

-        buf = strndup(payload, size);

-        if (!buf)

-                return -ENOMEM;

-

-        p = buf;

-        for (;;) {

-                _cleanup_free_ char *str;

-                union json_value v = {};

-                int t;

-

-                t = json_tokenize(&p, &str, &v, &json_state, NULL);

-                if (t < 0)

-                        return t;

-

-                switch (state) {

-

-                case STATE_BEGIN:

-                        if (t == JSON_ARRAY_OPEN)

-                                state = STATE_ITEM;

-                        else

-                                return -EBADMSG;

-

-                        break;

-

-                case STATE_ITEM:

-                        if (t == JSON_STRING) {

-                                if (!dkr_id_is_valid(str))

-                                        return -EBADMSG;

-

-                                if (n+1 > LAYERS_MAX)

-                                        return -EFBIG;

-

-                                if (!GREEDY_REALLOC(l, allocated, n + 2))

-                                        return -ENOMEM;

-

-                                l[n++] = str;

-                                str = NULL;

-                                l[n] = NULL;

-

-                                state = STATE_COMMA;

-

-                        } else if (t == JSON_ARRAY_CLOSE)

-                                state = STATE_END;

-                        else

-                                return -EBADMSG;

-

-                        break;

-

-                case STATE_COMMA:

-                        if (t == JSON_COMMA)

-                                state = STATE_ITEM;

-                        else if (t == JSON_ARRAY_CLOSE)

-                                state = STATE_END;

-                        else

-                                return -EBADMSG;

-                        break;

-

-                case STATE_END:

-                        if (t == JSON_END) {

-

-                                if (strv_isempty(l))

-                                        return -EBADMSG;

-

-                                if (!strv_is_uniq(l))

-                                        return -EBADMSG;

-

-                                l = strv_reverse(l);

-

-                                *ret = l;

-                                l = NULL;

-                                return 0;

-                        } else

-                                return -EBADMSG;

-                }

-

-        }

-}

-

-static const char *dkr_import_current_layer(DkrImport *i) {

-        assert(i);

-

-        if (strv_isempty(i->ancestry))

-                return NULL;

-

-        return i->ancestry[i->current_ancestry];

-}

-

-static const char *dkr_import_current_base_layer(DkrImport *i) {

-        assert(i);

-

-        if (strv_isempty(i->ancestry))

-                return NULL;

-

-        if (i->current_ancestry <= 0)

-                return NULL;

-

-        return i->ancestry[i->current_ancestry-1];

-}

-

-static int dkr_import_add_token(DkrImport *i, ImportJob *j) {

-        const char *t;

-

-        assert(i);

-        assert(j);

-

-        if (i->response_token)

-                t = strjoina("Authorization: Token ", i->response_token);

-        else

-                t = HEADER_TOKEN " true";

-

-        j->request_header = curl_slist_new("Accept: application/json", t, NULL);

-        if (!j->request_header)

-                return -ENOMEM;

-

-        return 0;

-}

-

-static bool dkr_import_is_done(DkrImport *i) {

-        assert(i);

-        assert(i->images_job);

-

-        if (i->images_job->state != IMPORT_JOB_DONE)

-                return false;

-

-        if (!i->tags_job || i->tags_job->state != IMPORT_JOB_DONE)

-                return false;

-

-        if (!i->ancestry_job || i->ancestry_job->state != IMPORT_JOB_DONE)

-                return false;

-

-        if (!i->json_job || i->json_job->state != IMPORT_JOB_DONE)

-                return false;

-

-        if (i->layer_job && i->layer_job->state != IMPORT_JOB_DONE)

-                return false;

-

-        if (dkr_import_current_layer(i))

-                return false;

-

-        return true;

-}

-

-static int dkr_import_make_local_copy(DkrImport *i) {

-        int r;

-

-        assert(i);

-

-        if (!i->local)

-                return 0;

-

-        if (!i->final_path) {

-                i->final_path = strjoin(i->image_root, "/.dkr-", i->id, NULL);

-                if (!i->final_path)

-                        return log_oom();

-        }

-

-        r = import_make_local_copy(i->final_path, i->image_root, i->local, i->force_local);

-        if (r < 0)

-                return r;

-

-        return 0;

-}

-

-static int dkr_import_job_on_open_disk(ImportJob *j) {

-        const char *base;

-        DkrImport *i;

-        int r;

-

-        assert(j);

-        assert(j->userdata);

-

-        i = j->userdata;

-        assert(i->layer_job == j);

-        assert(i->final_path);

-        assert(!i->temp_path);

-        assert(i->tar_pid <= 0);

-

-        r = tempfn_random(i->final_path, &i->temp_path);

-        if (r < 0)

-                return log_oom();

-

-        mkdir_parents_label(i->temp_path, 0700);

-

-        base = dkr_import_current_base_layer(i);

-        if (base) {

-                const char *base_path;

-

-                base_path = strjoina(i->image_root, "/.dkr-", base);

-                r = btrfs_subvol_snapshot(base_path, i->temp_path, false, true);

-        } else

-                r = btrfs_subvol_make(i->temp_path);

-        if (r < 0)

-                return log_error_errno(r, "Failed to make btrfs subvolume %s: %m", i->temp_path);

-

-        j->disk_fd = import_fork_tar(i->temp_path, &i->tar_pid);

-        if (j->disk_fd < 0)

-                return j->disk_fd;

-

-        return 0;

-}

-

-static void dkr_import_job_on_progress(ImportJob *j) {

-        DkrImport *i;

-

-        assert(j);

-        assert(j->userdata);

-

-        i = j->userdata;

-

-        dkr_import_report_progress(

-                        i,

-                        j == i->images_job                       ? DKR_SEARCHING :

-                        j == i->tags_job                         ? DKR_RESOLVING :

-                        j == i->ancestry_job || j == i->json_job ? DKR_METADATA :

-                                                                   DKR_DOWNLOADING);

-}

-

-static int dkr_import_pull_layer(DkrImport *i) {

-        _cleanup_free_ char *path = NULL;

-        const char *url, *layer = NULL;

-        int r;

-

-        assert(i);

-        assert(!i->layer_job);

-        assert(!i->temp_path);

-        assert(!i->final_path);

-

-        for (;;) {

-                layer = dkr_import_current_layer(i);

-                if (!layer)

-                        return 0; /* no more layers */

-

-                path = strjoin(i->image_root, "/.dkr-", layer, NULL);

-                if (!path)

-                        return log_oom();

-

-                if (laccess(path, F_OK) < 0) {

-                        if (errno == ENOENT)

-                                break;

-

-                        return log_error_errno(errno, "Failed to check for container: %m");

-                }

-

-                log_info("Layer %s already exists, skipping.", layer);

-

-                i->current_ancestry++;

-

-                free(path);

-                path = NULL;