Add support for Jupyter in GitLab via Kubernetes

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>

app/assets/javascripts/clusters/clusters_bundle.js

@@ -31,6 +31,7 @@ export default class Clusters {
       installHelmPath,
       installIngressPath,
       installRunnerPath,
+      installJupyterPath,
       installPrometheusPath,
       managePrometheusPath,
       clusterStatus,
@@ -51,6 +52,7 @@ export default class Clusters {
       installIngressEndpoint: installIngressPath,
       installRunnerEndpoint: installRunnerPath,
       installPrometheusEndpoint: installPrometheusPath,
+      installJupyterEndpoint: installJupyterPath,
     });
 
     this.installApplication = this.installApplication.bind(this);

app/assets/javascripts/clusters/components/applications.vue

@@ -37,6 +37,11 @@ export default {
       default: '',
     },
   },
+  data() {
+    return {
+      jupyterSuggestHostnameValue: '',
+    };
+  },
   computed: {
     generalApplicationDescription() {
       return sprintf(
@@ -121,6 +126,20 @@ export default {
         false,
       );
     },
+    jupyterInstalled() {
+      return this.applications.jupyter.status === APPLICATION_INSTALLED;
+    },
+    jupyterHostname() {
+      return this.applications.jupyter.hostname;
+    },
+    jupyterSuggestHostname() {
+      return `jupyter.${this.applications.ingress.externalIp}.xip.io`;
+    },
+  },
+  watch: {
+    jupyterSuggestHostname() {
+      this.jupyterSuggestHostnameValue = this.jupyterSuggestHostname;
+    },
   },
 };
 </script>
@@ -278,11 +297,89 @@ export default {
               applications to production.`) }}
           </div>
         </application-row>
+        <application-row
+          id="jupyter"
+          :title="applications.jupyter.title"
+          title-link="https://jupyterhub.readthedocs.io/en/stable/"
+          :status="applications.jupyter.status"
+          :status-reason="applications.jupyter.statusReason"
+          :request-status="applications.jupyter.requestStatus"
+          :request-reason="applications.jupyter.requestReason"
+        >
+          <div slot="description">
+            <p>
+              {{ s__(`ClusterIntegration|JupyterHub, a multi-user Hub, spawns,
+                manages, and proxies multiple instances of the single-user
+                Jupyter notebook server. JupyterHub can be used to serve
+                notebooks to a class of students, a corporate data science group,
+                or a scientific research group.`) }}
+            </p>
+            <template v-if="jupyterInstalled">
+              <div class="form-group">
+                <label for="jupyter-hostname">
+                  {{ s__('ClusterIntegration|Jupyter Hostname') }}
+                </label>
+                <div
+                  v-if="jupyterHostname"
+                  class="input-group"
+                >
+                  <input
+                    type="text"
+                    id="jupyter-hostname"
+                    class="form-control js-hostname"
+                    :value="jupyterHostname"
+                    readonly
+                  />
+                  <span class="input-group-btn">
+                    <clipboard-button
+                      :text="jupyterHostname"
+                      :title="s__('ClusterIntegration|Copy Jupyter Hostname to clipboard')"
+                      class="js-clipboard-btn"
+                    />
+                  </span>
+                </div>
+              </div>
+            </template>
+            <template v-else-if="ingressInstalled">
+              <div class="form-group">
+                <label for="jupyter-hostname">
+                  {{ s__('ClusterIntegration|Jupyter Hostname') }}
+                </label>
+                <div class="input-group">
+                  <input
+                    type="text"
+                    id="jupyter-hostname"
+                    class="form-control js-hostname"
+                    v-model="jupyterSuggestHostnameValue"
+                  />
+                  <span class="input-group-btn">
+                    <clipboard-button
+                      :text="jupyterHostname"
+                      :title="s__('ClusterIntegration|Copy Jupyter Hostname to clipboard')"
+                      class="js-clipboard-btn"
+                    />
+                  </span>
+                </div>
+              </div>
+              <p>
+                {{ s__(`ClusterIntegration|Replace this with your own hostname if you want.
+                If you do so, point hostname to Ingress IP Address from above.`) }}
+                <a
+                  :href="ingressDnsHelpPath"
+                  target="_blank"
+                  rel="noopener noreferrer"
+                >
+                  {{ __('More information') }}
+                </a>
+              </p>
+            </template>
+          </div>
+        </application-row>
         <!--
           NOTE: Don't forget to update `clusters.scss`
           min-height for this block and uncomment `application_spec` tests
         -->
-        <!-- Add GitLab Runner row, all other plumbing is complete -->
+        <!-- Add Jupyter row, all other plumbing is complete -->
       </div>
     </div>
   </section>

app/assets/javascripts/clusters/constants.js

@@ -11,3 +11,4 @@ export const REQUEST_LOADING = 'request-loading';
 export const REQUEST_SUCCESS = 'request-success';
 export const REQUEST_FAILURE = 'request-failure';
 export const INGRESS = 'ingress';
+export const JUPYTER = 'jupyter';

app/assets/javascripts/clusters/services/clusters_service.js

 import axios from '../../lib/utils/axios_utils';
+import { JUPYTER } from '../constants';
 
 export default class ClusterService {
   constructor(options = {}) {
@@ -8,6 +9,7 @@ export default class ClusterService {
       ingress: this.options.installIngressEndpoint,
       runner: this.options.installRunnerEndpoint,
       prometheus: this.options.installPrometheusEndpoint,
+      jupyter: this.options.installJupyterEndpoint,
     };
   }
 
@@ -16,7 +18,13 @@ export default class ClusterService {
   }
 
   installApplication(appId) {
-    return axios.post(this.appInstallEndpointMap[appId]);
+    const data = {};
+
+    if (appId === JUPYTER) {
+      data.hostname = document.getElementById('jupyter-hostname').value;
+    }
+
+    return axios.post(this.appInstallEndpointMap[appId], data);
   }
 
   static updateCluster(endpoint, data) {

app/assets/javascripts/clusters/stores/clusters_store.js

 import { s__ } from '../../locale';
-import { INGRESS } from '../constants';
+import { INGRESS, JUPYTER } from '../constants';
 
 export default class ClusterStore {
   constructor() {
@@ -38,6 +38,14 @@ export default class ClusterStore {
           requestStatus: null,
           requestReason: null,
         },
+        jupyter: {
+          title: s__('ClusterIntegration|JupyterHub'),
+          status: null,
+          statusReason: null,
+          requestStatus: null,
+          requestReason: null,
+          hostname: null,
+        },
       },
     };
   }
@@ -83,6 +91,8 @@ export default class ClusterStore {
 
       if (appId === INGRESS) {
         this.state.applications.ingress.externalIp = serverAppEntry.external_ip;
+      } else if (appId === JUPYTER) {
+        this.state.applications.jupyter.hostname = serverAppEntry.hostname;
       }
     });
   }

app/assets/stylesheets/pages/clusters.scss

@@ -6,7 +6,7 @@
 
 .cluster-applications-table {
   // Wait for the Vue to kick-in and render the applications block
-  min-height: 400px;
+  min-height: 500px;
 }
 
 .clusters-dropdown-menu {

app/controllers/projects/clusters/applications_controller.rb

@@ -6,6 +6,7 @@ class Projects::Clusters::ApplicationsController < Projects::ApplicationControll
 
   def create
     application = @application_class.find_or_create_by!(cluster: @cluster)
+    application.update(hostname: params[:hostname]) if application.respond_to?(:hostname)
 
     Clusters::Applications::ScheduleInstallationService.new(project, current_user).execute(application)
 

app/models/clusters/applications/jupyter.rb

@@ -12,17 +12,39 @@ module Clusters
       default_value_for :version, VERSION
 
       def chart
-        # TODO: publish jupyterhub charts that we can use for our installation
-        # and provide path to it here.
+        "#{name}/jupyterhub"
+      end
+
+      def repository
+        'https://jupyterhub.github.io/helm-chart/'
+      end
+
+      def values
+        content_values.to_yaml
       end
 
       def install_command
         Gitlab::Kubernetes::Helm::InstallCommand.new(
           name,
           chart: chart,
-          values: values
+          values: values,
+          repository: repository
         )
       end
+
+      private
+
+      def specification
+        {
+          "ingress" => { "hosts" => [hostname] },
+          "hub" => { "cookieSecret" => SecureRandom.hex(32) },
+          "proxy" => { "secretToken" => SecureRandom.hex(32) }
+        }
+      end
+
+      def content_values
+        YAML.load_file(chart_values_file).deep_merge!(specification)
+      end
     end
   end
 end

app/models/clusters/cluster.rb

@@ -27,6 +27,7 @@ module Clusters
     has_one :application_ingress, class_name: 'Clusters::Applications::Ingress'
     has_one :application_prometheus, class_name: 'Clusters::Applications::Prometheus'
     has_one :application_runner, class_name: 'Clusters::Applications::Runner'
+    has_one :application_jupyter, class_name: 'Clusters::Applications::Jupyter'
 
     accepts_nested_attributes_for :provider_gcp, update_only: true
     accepts_nested_attributes_for :platform_kubernetes, update_only: true
@@ -75,7 +76,8 @@ module Clusters
         application_helm || build_application_helm,
         application_ingress || build_application_ingress,
         application_prometheus || build_application_prometheus,
-        application_runner || build_application_runner
+        application_runner || build_application_runner,
+        application_jupyter || build_application_jupyter
       ]
     end
 

app/serializers/cluster_application_entity.rb

@@ -3,4 +3,5 @@ class ClusterApplicationEntity < Grape::Entity
   expose :status_name, as: :status
   expose :status_reason
   expose :external_ip, if: -> (e, _) { e.respond_to?(:external_ip) }
+  expose :hostname, if: -> (e, _) { e.respond_to?(:hostname) }
 end

app/services/clusters/applications/install_service.rb

@@ -12,8 +12,8 @@ module Clusters
             ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
         rescue Kubeclient::HttpError => ke
           app.make_errored!("Kubernetes error: #{ke.message}")
-        rescue StandardError
-          app.make_errored!("Can't start installation process")
+        rescue StandardError => e
+          app.make_errored!("Can't start installation process. #{e.message}")
         end
       end
     end

app/views/projects/clusters/show.html.haml

@@ -11,6 +11,7 @@
   install_ingress_path: install_applications_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster, :ingress),
   install_prometheus_path: install_applications_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster, :prometheus),
   install_runner_path: install_applications_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster, :runner),
+  install_jupyter_path: install_applications_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster, :jupyter),
   toggle_status: @cluster.enabled? ? 'true': 'false',
   cluster_status: @cluster.status_name,
   cluster_status_reason: @cluster.status_reason,

db/migrate/20180511131058_create_clusters_applications_jupyter.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CreateClustersApplicationsJupyter < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :clusters_applications_jupyters do |t|
+      t.references :cluster, null: false, unique: true, foreign_key: { on_delete: :cascade }
+
+      t.integer :status, null: false
+      t.string :version, null: false
+      t.string :hostname
+
+      t.text :status_reason
+
+      t.timestamps_with_timezone null: false
+    end
+  end
+end

spec/factories/clusters/applications/helm.rb

@@ -35,5 +35,6 @@ FactoryBot.define do
     factory :clusters_applications_ingress, class: Clusters::Applications::Ingress
     factory :clusters_applications_prometheus, class: Clusters::Applications::Prometheus
     factory :clusters_applications_runner, class: Clusters::Applications::Runner
+    factory :clusters_applications_jupyter, class: Clusters::Applications::Jupyter
   end
 end

spec/fixtures/api/schemas/cluster_status.json

@@ -31,7 +31,8 @@
           }
         },
         "status_reason": { "type": ["string", "null"] },
-        "external_ip": { "type": ["string", "null"] }
+        "external_ip": { "type": ["string", "null"] },
+        "hostname": { "type": ["string", "null"] }
       },
       "required" : [ "name", "status" ]
     }

spec/models/clusters/cluster_spec.rb

@@ -234,9 +234,10 @@ describe Clusters::Cluster do
       let!(:ingress) { create(:clusters_applications_ingress, cluster: cluster) }
       let!(:prometheus) { create(:clusters_applications_prometheus, cluster: cluster) }
       let!(:runner) { create(:clusters_applications_runner, cluster: cluster) }
+      let!(:jupyter) { create(:clusters_applications_jupyter, cluster: cluster) }
 
       it 'returns a list of created applications' do
-        is_expected.to contain_exactly(helm, ingress, prometheus, runner)
+        is_expected.to contain_exactly(helm, ingress, prometheus, runner, jupyter)
       end
     end
   end

vendor/jupyter/values.yaml

+rbac:
+  enabled: false
+
+hub:
+  extraEnv:
+    JUPYTER_ENABLE_LAB: 1
+  extraConfig: |
+    c.KubeSpawner.cmd = ['jupyter-labhub']
+
+singleuser:
+  defaultUrl: "/lab"
+
+ingress:
+ enabled: true
+ annotations:
+   kubernetes.io/ingress.class: "nginx"