Merge branch '276949-link-perf-prom' into 'master'

Pipeline Graph Structural Update: Add Link Perf

See merge request gitlab-org/gitlab!53902

app/assets/javascripts/performance/constants.js

@@ -54,3 +54,24 @@ export const MR_DIFFS_MARK_DIFF_FILES_END = 'mr-diffs-mark-diff-files-end';
 // Measures
 export const MR_DIFFS_MEASURE_FILE_TREE_DONE = 'mr-diffs-measure-file-tree-done';
 export const MR_DIFFS_MEASURE_DIFF_FILES_DONE = 'mr-diffs-measure-diff-files-done';
+
+//
+// Pipelines Detail namespace
+//
+
+// Marks
+export const PIPELINES_DETAIL_LINKS_MARK_CALCULATE_START =
+  'pipelines-detail-links-mark-calculate-start';
+export const PIPELINES_DETAIL_LINKS_MARK_CALCULATE_END =
+  'pipelines-detail-links-mark-calculate-end';
+
+// Measures
+export const PIPELINES_DETAIL_LINKS_MEASURE_CALCULATION =
+  'Pipelines Detail Graph: Links Calculation';
+
+// Metrics
+// Note: These strings must match the backend
+// (defined in: app/services/ci/prometheus_metrics/observe_histograms_service.rb)
+export const PIPELINES_DETAIL_LINK_DURATION = 'pipeline_graph_link_calculation_duration_seconds';
+export const PIPELINES_DETAIL_LINKS_TOTAL = 'pipeline_graph_links_total';
+export const PIPELINES_DETAIL_LINKS_JOB_RATIO = 'pipeline_graph_link_per_job_ratio';

app/assets/javascripts/pipelines/components/graph/graph_component.vue

@@ -15,14 +15,19 @@ export default {
     StageColumnComponent,
   },
   props: {
+    pipeline: {
+      type: Object,
+      required: true,
+    },
     isLinkedPipeline: {
       type: Boolean,
       required: false,
       default: false,
     },
-    pipeline: {
-      type: Object,
-      required: true,
+    metricsPath: {
+      type: String,
+      required: false,
+      default: '',
     },
     type: {
       type: String,
@@ -66,6 +71,12 @@ export default {
     hasUpstreamPipelines() {
       return Boolean(this.pipeline?.upstream?.length > 0);
     },
+    metricsConfig() {
+      return {
+        path: this.metricsPath,
+        collectMetrics: true,
+      };
+    },
     // The show downstream check prevents showing redundant linked columns
     showDownstreamPipelines() {
       return (
@@ -145,6 +156,7 @@ export default {
               :container-id="containerId"
               :container-measurements="measurements"
               :highlighted-job="hoveredJobName"
+              :metrics-config="metricsConfig"
               default-link-color="gl-stroke-transparent"
               @error="onError"
               @highlightedJobsChange="updateHighlightedJobs"

app/assets/javascripts/pipelines/components/graph/graph_component_wrapper.vue

@@ -14,6 +14,9 @@ export default {
     PipelineGraph,
   },
   inject: {
+    metricsPath: {
+      default: '',
+    },
     pipelineIid: {
       default: '',
     },
@@ -108,6 +111,7 @@ export default {
     <gl-loading-icon v-if="showLoadingIcon" class="gl-mx-auto gl-my-4" size="lg" />
     <pipeline-graph
       v-if="pipeline"
+      :metrics-path="metricsPath"
       :pipeline="pipeline"
       @error="reportFailure"
       @refreshPipelineGraph="refreshPipelineGraph"

app/assets/javascripts/pipelines/components/graph_shared/api.js

+import axios from '~/lib/utils/axios_utils';
+import { reportToSentry } from '../graph/utils';
+
+export const reportPerformance = (path, stats) => {
+  axios.post(path, stats).catch((err) => {
+    reportToSentry('links_inner_perf', `error: ${err}`);
+  });
+};

app/assets/javascripts/pipelines/components/graph_shared/links_inner.vue

 <script>
 import { isEmpty } from 'lodash';
+import {
+  PIPELINES_DETAIL_LINKS_MARK_CALCULATE_START,
+  PIPELINES_DETAIL_LINKS_MARK_CALCULATE_END,
+  PIPELINES_DETAIL_LINKS_MEASURE_CALCULATION,
+  PIPELINES_DETAIL_LINK_DURATION,
+  PIPELINES_DETAIL_LINKS_TOTAL,
+  PIPELINES_DETAIL_LINKS_JOB_RATIO,
+} from '~/performance/constants';
+import { performanceMarkAndMeasure } from '~/performance/utils';
 import { DRAW_FAILURE } from '../../constants';
 import { createJobsHash, generateJobNeedsDict } from '../../utils';
 import { reportToSentry } from '../graph/utils';
 import { parseData } from '../parsing_utils';
+import { reportPerformance } from './api';
 import { generateLinksData } from './drawing_utils';
 
 export default {
@@ -26,6 +36,15 @@ export default {
       type: Array,
       required: true,
     },
+    totalGroups: {
+      type: Number,
+      required: true,
+    },
+    metricsConfig: {
+      type: Object,
+      required: false,
+      default: () => ({}),
+    },
     defaultLinkColor: {
       type: String,
       required: false,
@@ -44,6 +63,9 @@ export default {
     };
   },
   computed: {
+    shouldCollectMetrics() {
+      return this.metricsConfig.collectMetrics && this.metricsConfig.path;
+    },
     hasHighlightedJob() {
       return Boolean(this.highlightedJob);
     },
@@ -97,10 +119,52 @@ export default {
     }
   },
   methods: {
+    beginPerfMeasure() {
+      if (this.shouldCollectMetrics) {
+        performanceMarkAndMeasure({ mark: PIPELINES_DETAIL_LINKS_MARK_CALCULATE_START });
+      }
+    },
+    finishPerfMeasureAndSend() {
+      if (this.shouldCollectMetrics) {
+        performanceMarkAndMeasure({
+          mark: PIPELINES_DETAIL_LINKS_MARK_CALCULATE_END,
+          measures: [
+            {
+              name: PIPELINES_DETAIL_LINKS_MEASURE_CALCULATION,
+              start: PIPELINES_DETAIL_LINKS_MARK_CALCULATE_START,
+            },
+          ],
+        });
+      }
+
+      window.requestAnimationFrame(() => {
+        const duration = window.performance.getEntriesByName(
+          PIPELINES_DETAIL_LINKS_MEASURE_CALCULATION,
+        )[0]?.duration;
+
+        if (!duration) {
+          return;
+        }
+
+        const data = {
+          histograms: [
+            { name: PIPELINES_DETAIL_LINK_DURATION, value: duration },
+            { name: PIPELINES_DETAIL_LINKS_TOTAL, value: this.links.length },
+            {
+              name: PIPELINES_DETAIL_LINKS_JOB_RATIO,
+              value: this.links.length / this.totalGroups,
+            },
+          ],
+        };
+
+        reportPerformance(this.metricsConfig.path, data);
+      });
+    },
     isLinkHighlighted(linkRef) {
       return this.highlightedLinks.includes(linkRef);
     },
     prepareLinkData() {
+      this.beginPerfMeasure();
       try {
         const arrayOfJobs = this.pipelineData.flatMap(({ groups }) => groups);
         const parsedData = parseData(arrayOfJobs);
@@ -109,6 +173,7 @@ export default {
         this.$emit('error', DRAW_FAILURE);
         reportToSentry(this.$options.name, err);
       }
+      this.finishPerfMeasureAndSend();
     },
     getLinkClasses(link) {
       return [

app/assets/javascripts/pipelines/components/graph_shared/links_layer.vue

@@ -70,6 +70,7 @@ export default {
     v-if="showLinkedLayers"
     :container-measurements="containerMeasurements"
     :pipeline-data="pipelineData"
+    :total-groups="numGroups"
     v-bind="$attrs"
     v-on="$listeners"
   >

app/assets/javascripts/pipelines/pipeline_details_bundle.js

@@ -93,8 +93,13 @@ export default async function initPipelineDetailsBundle() {
         /* webpackChunkName: 'createPipelinesDetailApp' */ './pipeline_details_graph'
       );
 
-      const { pipelineProjectPath, pipelineIid } = dataset;
-      createPipelinesDetailApp(SELECTORS.PIPELINE_GRAPH, pipelineProjectPath, pipelineIid);
+      const { metricsPath, pipelineProjectPath, pipelineIid } = dataset;
+      createPipelinesDetailApp(
+        SELECTORS.PIPELINE_GRAPH,
+        pipelineProjectPath,
+        pipelineIid,
+        metricsPath,
+      );
     } catch {
       Flash(__('An error occurred while loading the pipeline.'));
     }

app/assets/javascripts/pipelines/pipeline_details_graph.js

@@ -16,7 +16,7 @@ const apolloProvider = new VueApollo({
   ),
 });
 
-const createPipelinesDetailApp = (selector, pipelineProjectPath, pipelineIid) => {
+const createPipelinesDetailApp = (selector, pipelineProjectPath, pipelineIid, metricsPath) => {
   // eslint-disable-next-line no-new
   new Vue({
     el: selector,
@@ -25,6 +25,7 @@ const createPipelinesDetailApp = (selector, pipelineProjectPath, pipelineIid) =>
     },
     apolloProvider,
     provide: {
+      metricsPath,
       pipelineProjectPath,
       pipelineIid,
       dataMethod: GRAPHQL,

app/views/projects/pipelines/show.html.haml

@@ -26,4 +26,4 @@
 
   = render "projects/pipelines/with_tabs", pipeline: @pipeline, pipeline_has_errors: pipeline_has_errors
 
-.js-pipeline-details-vue{ data: { endpoint: project_pipeline_path(@project, @pipeline, format: :json), pipeline_project_path: @project.full_path, pipeline_iid: @pipeline.iid } }
+.js-pipeline-details-vue{ data: { endpoint: project_pipeline_path(@project, @pipeline, format: :json), metrics_path: namespace_project_ci_prometheus_metrics_histograms_path(namespace_id: @project.namespace, project_id: @project, format: :json), pipeline_project_path: @project.full_path, pipeline_iid: @pipeline.iid } }

spec/frontend/pipelines/graph_shared/links_inner_spec.js

 import { shallowMount } from '@vue/test-utils';
+import MockAdapter from 'axios-mock-adapter';
 import { setHTMLFixture } from 'helpers/fixtures';
+import axios from '~/lib/utils/axios_utils';
+import {
+  PIPELINES_DETAIL_LINK_DURATION,
+  PIPELINES_DETAIL_LINKS_TOTAL,
+  PIPELINES_DETAIL_LINKS_JOB_RATIO,
+} from '~/performance/constants';
+import * as perfUtils from '~/performance/utils';
+import * as sentryUtils from '~/pipelines/components/graph/utils';
+import * as Api from '~/pipelines/components/graph_shared/api';
 import LinksInner from '~/pipelines/components/graph_shared/links_inner.vue';
 import { createJobsHash } from '~/pipelines/utils';
 import {
@@ -18,7 +28,9 @@ describe('Links Inner component', () => {
     containerMeasurements: { width: 1019, height: 445 },
     pipelineId: 1,
     pipelineData: [],
+    totalGroups: 10,
   };
+
   let wrapper;
 
   const createComponent = (props) => {
@@ -194,4 +206,141 @@ describe('Links Inner component', () => {
       expect(firstLink.classes(hoverColorClass)).toBe(true);
     });
   });
+
+  describe('performance metrics', () => {
+    let markAndMeasure;
+    let reportToSentry;
+    let reportPerformance;
+    let mock;
+
+    beforeEach(() => {
+      mock = new MockAdapter(axios);
+      jest.spyOn(window, 'requestAnimationFrame').mockImplementation((cb) => cb());
+      markAndMeasure = jest.spyOn(perfUtils, 'performanceMarkAndMeasure');
+      reportToSentry = jest.spyOn(sentryUtils, 'reportToSentry');
+      reportPerformance = jest.spyOn(Api, 'reportPerformance');
+    });
+
+    afterEach(() => {
+      mock.restore();
+    });
+
+    describe('with no metrics config object', () => {
+      beforeEach(() => {
+        setFixtures(pipelineData);
+        createComponent({
+          pipelineData: pipelineData.stages,
+        });
+      });
+
+      it('is not called', () => {
+        expect(markAndMeasure).not.toHaveBeenCalled();
+        expect(reportToSentry).not.toHaveBeenCalled();
+        expect(reportPerformance).not.toHaveBeenCalled();
+      });
+    });
+
+    describe('with metrics config set to false', () => {
+      beforeEach(() => {
+        setFixtures(pipelineData);
+        createComponent({
+          pipelineData: pipelineData.stages,
+          metricsConfig: {
+            collectMetrics: false,
+            metricsPath: '/path/to/metrics',
+          },
+        });
+      });
+
+      it('is not called', () => {
+        expect(markAndMeasure).not.toHaveBeenCalled();
+        expect(reportToSentry).not.toHaveBeenCalled();
+        expect(reportPerformance).not.toHaveBeenCalled();
+      });
+    });
+
+    describe('with no metrics path', () => {
+      beforeEach(() => {
+        setFixtures(pipelineData);
+        createComponent({
+          pipelineData: pipelineData.stages,
+          metricsConfig: {
+            collectMetrics: true,
+            metricsPath: '',
+          },
+        });
+      });
+
+      it('is not called', () => {
+        expect(markAndMeasure).not.toHaveBeenCalled();
+        expect(reportToSentry).not.toHaveBeenCalled();
+        expect(reportPerformance).not.toHaveBeenCalled();
+      });
+    });
+
+    describe('with metrics path and collect set to true', () => {
+      const metricsPath = '/root/project/-/ci/prometheus_metrics/histograms.json';
+      const duration = 0.0478;
+      const numLinks = 1;
+      const metricsData = {
+        histograms: [
+          { name: PIPELINES_DETAIL_LINK_DURATION, value: duration },
+          { name: PIPELINES_DETAIL_LINKS_TOTAL, value: numLinks },
+          {
+            name: PIPELINES_DETAIL_LINKS_JOB_RATIO,
+            value: numLinks / defaultProps.totalGroups,
+          },
+        ],
+      };
+
+      describe('when no duration is obtained', () => {
+        beforeEach(() => {
+          jest.spyOn(window.performance, 'getEntriesByName').mockImplementation(() => {
+            return [];
+          });
+
+          setFixtures(pipelineData);
+
+          createComponent({
+            pipelineData: pipelineData.stages,
+            metricsConfig: {
+              collectMetrics: true,
+              path: metricsPath,
+            },
+          });
+        });
+
+        it('attempts to collect metrics', () => {
+          expect(markAndMeasure).toHaveBeenCalled();
+          expect(reportPerformance).not.toHaveBeenCalled();
+          expect(reportToSentry).not.toHaveBeenCalled();
+        });
+      });
+
+      describe('with duration and no error', () => {
+        beforeEach(() => {
+          jest.spyOn(window.performance, 'getEntriesByName').mockImplementation(() => {
+            return [{ duration }];
+          });
+
+          setFixtures(pipelineData);
+
+          createComponent({
+            pipelineData: pipelineData.stages,
+            metricsConfig: {
+              collectMetrics: true,
+              path: metricsPath,
+            },
+          });
+        });
+
+        it('it calls reportPerformance with expected arguments', () => {
+          expect(markAndMeasure).toHaveBeenCalled();
+          expect(reportPerformance).toHaveBeenCalled();
+          expect(reportPerformance).toHaveBeenCalledWith(metricsPath, metricsData);
+          expect(reportToSentry).not.toHaveBeenCalled();
+        });
+      });
+    });
+  });
 });