Add support for bypass_container_update param for easier local build and testing (#6702)

Signed-off-by: Alan Savage <3028205+asavageiv@users.noreply.github.com>
Signed-off-by: Simon L. <szaimen@e.mail.de>
Signed-off-by: Alan Savage <asavageiv@users.noreply.github.com>
Co-authored-by: Alan Savage <asavage@ifrit.internal>
Co-authored-by: Alan Savage <3028205+asavageiv@users.noreply.github.com>
Co-authored-by: Simon L. <szaimen@e.mail.de>

Containers/mastercontainer/Dockerfile

@@ -8,9 +8,6 @@ FROM caddy:2.10.0-alpine AS caddy
 # From https://github.com/docker-library/php/blob/master/8.4/alpine3.22/fpm/Dockerfile
 FROM php:8.4.11-fpm-alpine3.22
 
-ARG AIO_GIT_URL="https://github.com/nextcloud-releases/all-in-one.git"
-ARG AIO_GIT_BRANCH="main"
-
 EXPOSE 80
 EXPOSE 8080
 EXPOSE 8443
@@ -18,6 +15,13 @@ EXPOSE 8443
 COPY --from=caddy /usr/bin/caddy /usr/bin/caddy
 COPY --from=docker /usr/local/bin/docker /usr/local/bin/docker
 
+COPY community-containers /var/www/docker-aio/community-containers
+COPY php /var/www/docker-aio/php
+COPY --chmod=775 Containers/mastercontainer/*.sh /
+COPY --chmod=664 Containers/mastercontainer/Caddyfile /Caddyfile
+COPY --chmod=664 Containers/mastercontainer/supervisord.conf /supervisord.conf
+COPY Containers/mastercontainer/mastercontainer.conf /etc/apache2/sites-available/mastercontainer.conf
+
 WORKDIR /var/www/docker-aio
 
 # hadolint ignore=SC2086,DL3047,DL3003,DL3004
@@ -67,8 +71,6 @@ RUN set -ex; \
     wget https://getcomposer.org/installer -O - | php -- --install-dir=/usr/local/bin --filename=composer; \
     chmod +x /usr/local/bin/composer; \
     cd /var/www/docker-aio; \
-    git clone "$AIO_GIT_URL" --depth 1 --single-branch --branch "$AIO_GIT_BRANCH" .; \
-    find ./ -maxdepth 1 -mindepth 1 -not -path ./php -not -path ./community-containers -exec rm -r {} \; ; \
     rm -r ./php/tests; \
     chown www-data:www-data -R /var/www/docker-aio; \
     cd php; \
@@ -120,11 +122,6 @@ RUN set -ex; \
     mkdir /var/log/supervisord; \
     mkdir /var/run/supervisord;
 
-COPY --chmod=775 *.sh /
-COPY --chmod=664 Caddyfile /Caddyfile
-COPY --chmod=664 supervisord.conf /supervisord.conf
-COPY mastercontainer.conf /etc/apache2/sites-available/mastercontainer.conf
-
 LABEL org.label-schema.vendor="Nextcloud"
 
 # hadolint ignore=DL3002

Containers/nextcloud/Dockerfile

@@ -13,11 +13,15 @@ ENV AIO_TOKEN=123456
 ENV AIO_URL=localhost
 # AIO settings end # Do not remove or change this line!
 
-COPY --chmod=775 *.sh /
+COPY --chmod=775 Containers/nextcloud/*.sh /
-COPY --chmod=774 upgrade.exclude /upgrade.exclude
+COPY --chmod=774 Containers/nextcloud/upgrade.exclude /upgrade.exclude
-COPY config/*.php /
+COPY Containers/nextcloud/config/*.php /
-COPY supervisord.conf /supervisord.conf
+COPY Containers/nextcloud/supervisord.conf /supervisord.conf
-COPY root.motd /root.motd
+
+# AIO cloning start # Do not remove or change this line!
+COPY app /usr/src/nextcloud/apps/nextcloud-aio
+COPY Containers/nextcloud/root.motd /root.motd
+# AIO cloning end # Do not remove or change this line!
 
 VOLUME /mnt/ncdata
 VOLUME /var/www/html
@@ -242,15 +246,7 @@ RUN set -ex; \
     sed -i 's/^pm.max_children =.*/pm.max_children = 5000/' /usr/local/etc/php-fpm.d/www.conf; \
     sed -i 's|access.log = /proc/self/fd/2|access.log = /proc/self/fd/1|' /usr/local/etc/php-fpm.d/docker.conf; \
     \
-# AIO cloning start # Do not remove or change this line!
+    echo "[ -n \"\$TERM\" ] && [ -f /root.motd ] && cat /root.motd" >> /root/.bashrc; \
-    rm -rf /tmp/nextcloud-aio && \
-    mkdir -p /tmp/nextcloud-aio && \
-    cd /tmp/nextcloud-aio && \
-    git clone https://github.com/nextcloud-releases/all-in-one.git --depth 1 .; \
-    mkdir -p /usr/src/nextcloud/apps/nextcloud-aio; \
-    cp -r ./app/* /usr/src/nextcloud/apps/nextcloud-aio/; \
-    echo "[ -n \"\$TERM\" ] && cat /root.motd" >> /root/.bashrc; \
-# AIO cloning end # Do not remove or change this line!
     \
     chown www-data:root -R /usr/src && \
     chmod 777 -R /usr/local/etc/php/conf.d && \

develop.md

@@ -26,7 +26,7 @@ Simply use https://github.com/nextcloud/all-in-one/issues/6198 as template.
 Go to https://github.com/nextcloud-releases/all-in-one/actions/workflows/repo-sync.yml and run the workflow that will first sync the repo and then build new container that automatically get published to `develop` and `develop-arm64`.
 
 ## How to test things correctly?
-Before testing, make sure that at least the amd64 containers are built successfully by checking the last workflow here: https://github.com/nextcloud-releases/all-in-one/actions/workflows/build_images.yml. 
+Before testing, make sure that at least the amd64 containers are built successfully by checking the last workflow here: https://github.com/nextcloud-releases/all-in-one/actions/workflows/build_images.yml.
 
 There is a testing-VM available for the maintainer of AIO that allows for some final testing before releasing new version. See [this](https://cloud.nextcloud.com/apps/collectives/Nextcloud%20Handbook/Technical/AIO%20testing%20VM?fileId=6350152) for details.
 
@@ -48,12 +48,25 @@ This is documented here: https://github.com/nextcloud-releases/all-in-one/tree/m
 ## How to connect to the database?
 Simply run `sudo docker exec -it nextcloud-aio-database psql -U oc_nextcloud nextcloud_database` and you should be in.
 
-## How to locally build and test changes to mastercontainer?
+## How to locally build and test changes to mastercontainer
-1. Push changes to your own git fork and branch.
+1. Ensure you are on the developer channel per the instructions above.
-1. Use below commands to build mastercontainer image for a custom git url and branch:
+1. Use the commands below from the project root to build the mastercontainer image:
 ```
-cd Containers/mastercontainer
+docker buildx build --file Containers/mastercontainer/Dockerfile --tag ghcr.io/nextcloud-releases/all-in-one:develop --load .
-docker buildx build -t ghcr.io/nextcloud-releases/all-in-one:latest --build-arg AIO_GIT_URL="https://github.com/my-fork-repo/all-in-one.git" --build-arg AIO_GIT_BRANCH="my-feature-branch" --load .
 ```
 1. Start a container with above built image.
 1. Since the hash of a locally built image doesn't match the latest release mastercontainer, it prompts for a mandatory update. To temporarily bypass the update suffix `?bypass_mastercontainer_update` to the URL. Eg: `https://localhost:8080/containers?bypass_mastercontainer_update`
+
+## How to locally build and test changes to other containers using the bypass_container_update param
+1. Ensure you are on the developer channel per the instructions above.
+1. Use the commands below from the project root to build the container image:
+```
+# For the "nextcloud" container
+docker buildx build --file Containers/nextcloud/Dockerfile --tag ghcr.io/nextcloud-releases/aio-nextcloud:develop --load .
+
+# For all other containers
+docker buildx build --file Containers/{container}/Dockerfile --tag ghcr.io/nextcloud-releases/aio-{container}:develop --load Containers/{container}
+```
+1. Stop the containers using the AIO admin interface.
+1. Reload the AIO admin interface with the param `bypass_container_update` to avoid overwriting your local changes, e.g. `https://localhost:8080/containers?bypass_container_update`.
+1. Click "Start and update containers" and test your changes. Containers will not be updated, despite the button text.

php/public/index.php

@@ -86,6 +86,7 @@ $app->get('/containers', function (Request $request, Response $response, array $
     // Check if bypass_mastercontainer_update is provided on the URL, a special developer mode to bypass a mastercontainer update and use local image.
     $params = $request->getQueryParams();
     $bypass_mastercontainer_update = isset($params['bypass_mastercontainer_update']);
+    $bypass_container_update = isset($params['bypass_container_update']);
 
     return $view->render($response, 'containers.twig', [
         'domain' => $configurationManager->GetDomain(),
@@ -136,6 +137,7 @@ $app->get('/containers', function (Request $request, Response $response, array $
         'is_whiteboard_enabled' => $configurationManager->isWhiteboardEnabled(),
         'community_containers' => $configurationManager->listAvailableCommunityContainers(),
         'community_containers_enabled' => $configurationManager->GetEnabledCommunityContainers(),
+        'bypass_container_update' => $bypass_container_update,
     ]);
 })->setName('profile');
 $app->get('/login', function (Request $request, Response $response, array $args) use ($container) {

php/src/Controller/DockerController.php

@@ -191,8 +191,14 @@ readonly class DockerController {
         $config['install_latest_major'] = $installLatestMajor;
         $this->configurationManager->WriteConfig($config);
 
+        // Do not pull container images in case 'bypass_container_update' is set via url params
+        // Needed for local testing
+        $pullImage = !isset($request->getParsedBody()['bypass_container_update']);
+        if ($pullImage === false) {
+            error_log('WARNING: Not pulling container images. Instead, using local ones.');
+        }
         // Start container
-        $this->startTopContainer(true);
+        $this->startTopContainer($pullImage);
 
         // Clear apcu cache in order to check if container updates are available
         // Temporarily disabled as it leads much faster to docker rate limits

php/templates/containers.twig

@@ -352,6 +352,7 @@
                                     <form method="POST" action="/api/docker/start" class="xhr">
                                         <input type="hidden" name="{{csrf.keys.name}}" value="{{csrf.name}}">
                                         <input type="hidden" name="{{csrf.keys.value}}" value="{{csrf.value}}">
+                                        <input type="hidden" name="bypass_container_update" value="{{bypass_container_update}}">
                                         <input class="button " type="submit" value="Start and update containers" onclick="return confirm('Start and update containers? You should consider creating a backup first.')" />
                                     </form>
                                 {% endif %}