Commit 7d1d33ae authored by Peter Hegman's avatar Peter Hegman

Merge branch '337226-lazy-load-commit-data' into 'master'

Lazy load commit data

See merge request gitlab-org/gitlab!71633
parents 61025890 037f5eef
import axios from '~/lib/utils/axios_utils';
import { joinPaths } from '~/lib/utils/url_utility';
import { normalizeData } from 'ee_else_ce/repository/utils/commit';
import createFlash from '~/flash';
import { COMMIT_BATCH_SIZE, I18N_COMMIT_DATA_FETCH_ERROR } from './constants';
let requestedOffsets = [];
let fetchedBatches = [];
export const isRequested = (offset) => requestedOffsets.includes(offset);
export const resetRequestedCommits = () => {
requestedOffsets = [];
fetchedBatches = [];
};
const addRequestedOffset = (offset) => {
if (isRequested(offset) || offset < 0) {
return;
}
requestedOffsets.push(offset);
};
const removeLeadingSlash = (path) => path.replace(/^\//, '');
const fetchData = (projectPath, path, ref, offset) => {
if (fetchedBatches.includes(offset) || offset < 0) {
return [];
}
fetchedBatches.push(offset);
const url = joinPaths(
gon.relative_url_root || '/',
projectPath,
'/-/refs/',
ref,
'/logs_tree/',
encodeURIComponent(removeLeadingSlash(path)),
);
return axios
.get(url, { params: { format: 'json', offset } })
.then(({ data }) => normalizeData(data, path))
.catch(() => createFlash({ message: I18N_COMMIT_DATA_FETCH_ERROR }));
};
export const loadCommits = async (projectPath, path, ref, offset) => {
if (isRequested(offset)) {
return [];
}
// We fetch in batches of 25, so this ensures we don't refetch
Array.from(Array(COMMIT_BATCH_SIZE)).forEach((_, i) => {
addRequestedOffset(offset - i);
addRequestedOffset(offset + i);
});
// Since a user could scroll either up or down, we want to support lazy loading in both directions
const commitsBatchUp = await fetchData(projectPath, path, ref, offset - COMMIT_BATCH_SIZE);
const commitsBatchDown = await fetchData(projectPath, path, ref, offset);
return commitsBatchUp.concat(commitsBatchDown);
};
<script>
import { GlDeprecatedSkeletonLoading as GlSkeletonLoading, GlButton } from '@gitlab/ui';
import glFeatureFlagMixin from '~/vue_shared/mixins/gl_feature_flags_mixin';
import { sprintf, __ } from '../../../locale';
import getRefMixin from '../../mixins/get_ref';
import projectPathQuery from '../../queries/project_path.query.graphql';
......@@ -15,13 +16,18 @@ export default {
ParentRow,
GlButton,
},
mixins: [getRefMixin],
mixins: [getRefMixin, glFeatureFlagMixin()],
apollo: {
projectPath: {
query: projectPathQuery,
},
},
props: {
commits: {
type: Array,
required: false,
default: () => [],
},
path: {
type: String,
required: true,
......@@ -48,6 +54,7 @@ export default {
data() {
return {
projectPath: '',
rowNumbers: {},
};
},
computed: {
......@@ -73,10 +80,37 @@ export default {
return ['', '/'].indexOf(this.path) === -1;
},
},
watch: {
$route: function routeChange() {
this.$options.totalRowsLoaded = -1;
},
},
totalRowsLoaded: -1,
methods: {
showMore() {
this.$emit('showMore');
},
generateRowNumber(id) {
if (!this.glFeatures.lazyLoadCommits) {
return 0;
}
if (!this.rowNumbers[id] && this.rowNumbers[id] !== 0) {
this.$options.totalRowsLoaded += 1;
this.rowNumbers[id] = this.$options.totalRowsLoaded;
}
return this.rowNumbers[id];
},
getCommit(fileName, type) {
if (!this.glFeatures.lazyLoadCommits) {
return {};
}
return this.commits.find(
(commitEntry) => commitEntry.fileName === fileName && commitEntry.type === type,
);
},
},
};
</script>
......@@ -116,6 +150,9 @@ export default {
:lfs-oid="entry.lfsOid"
:loading-path="loadingPath"
:total-entries="totalEntries"
:row-number="generateRowNumber(entry.id)"
:commit-info="getCommit(entry.name, entry.type)"
v-on="$listeners"
/>
</template>
<template v-if="isLoading">
......
......@@ -8,6 +8,7 @@ import {
GlIcon,
GlHoverLoadDirective,
GlSafeHtmlDirective,
GlIntersectionObserver,
} from '@gitlab/ui';
import { escapeRegExp } from 'lodash';
import filesQuery from 'shared_queries/repository/files.query.graphql';
......@@ -30,6 +31,7 @@ export default {
GlIcon,
TimeagoTooltip,
FileIcon,
GlIntersectionObserver,
},
directives: {
GlTooltip: GlTooltipDirective,
......@@ -48,10 +50,23 @@ export default {
maxOffset: this.totalEntries,
};
},
skip() {
return this.glFeatures.lazyLoadCommits;
},
},
},
mixins: [getRefMixin, glFeatureFlagMixin()],
props: {
commitInfo: {
type: Object,
required: false,
default: null,
},
rowNumber: {
type: Number,
required: false,
default: null,
},
totalEntries: {
type: Number,
required: true,
......@@ -113,9 +128,13 @@ export default {
data() {
return {
commit: null,
hasRowAppeared: false,
};
},
computed: {
commitData() {
return this.glFeatures.lazyLoadCommits ? this.commitInfo : this.commit;
},
refactorBlobViewerEnabled() {
return this.glFeatures.refactorBlobViewer;
},
......@@ -148,7 +167,10 @@ export default {
return this.sha.slice(0, 8);
},
hasLockLabel() {
return this.commit && this.commit.lockLabel;
return this.commitData && this.commitData.lockLabel;
},
showSkeletonLoader() {
return !this.commitData && this.hasRowAppeared;
},
},
methods: {
......@@ -179,6 +201,19 @@ export default {
apolloQuery(query, variables) {
this.$apollo.query({ query, variables });
},
rowAppeared() {
this.hasRowAppeared = true;
if (this.glFeatures.lazyLoadCommits) {
this.$emit('row-appear', {
rowNumber: this.rowNumber,
hasCommit: Boolean(this.commitInfo),
});
}
},
rowDisappeared() {
this.hasRowAppeared = false;
},
},
safeHtmlConfig: { ADD_TAGS: ['gl-emoji'] },
};
......@@ -222,7 +257,7 @@ export default {
<gl-icon
v-if="hasLockLabel"
v-gl-tooltip
:title="commit.lockLabel"
:title="commitData.lockLabel"
name="lock"
:size="12"
class="ml-1"
......@@ -230,17 +265,19 @@ export default {
</td>
<td class="d-none d-sm-table-cell tree-commit cursor-default">
<gl-link
v-if="commit"
v-safe-html:[$options.safeHtmlConfig]="commit.titleHtml"
:href="commit.commitPath"
:title="commit.message"
v-if="commitData"
v-safe-html:[$options.safeHtmlConfig]="commitData.titleHtml"
:href="commitData.commitPath"
:title="commitData.message"
class="str-truncated-100 tree-commit-link"
/>
<gl-skeleton-loading v-else :lines="1" class="h-auto" />
<gl-intersection-observer @appear="rowAppeared" @disappear="rowDisappeared">
<gl-skeleton-loading v-if="showSkeletonLoader" :lines="1" class="h-auto" />
</gl-intersection-observer>
</td>
<td class="tree-time-ago text-right cursor-default">
<timeago-tooltip v-if="commit" :time="commit.committedDate" />
<gl-skeleton-loading v-else :lines="1" class="ml-auto h-auto w-50" />
<timeago-tooltip v-if="commitData" :time="commitData.committedDate" />
<gl-skeleton-loading v-if="showSkeletonLoader" :lines="1" class="ml-auto h-auto w-50" />
</td>
</tr>
</template>
......@@ -8,6 +8,7 @@ import { TREE_PAGE_SIZE, TREE_INITIAL_FETCH_COUNT, TREE_PAGE_LIMIT } from '../co
import getRefMixin from '../mixins/get_ref';
import projectPathQuery from '../queries/project_path.query.graphql';
import { readmeFile } from '../utils/readme';
import { loadCommits, isRequested, resetRequestedCommits } from '../commits_service';
import FilePreview from './preview/index.vue';
import FileTable from './table/index.vue';
......@@ -36,6 +37,7 @@ export default {
},
data() {
return {
commits: [],
projectPath: '',
nextPageCursor: '',
pagesLoaded: 1,
......@@ -81,12 +83,16 @@ export default {
this.entries.submodules = [];
this.entries.blobs = [];
this.nextPageCursor = '';
resetRequestedCommits();
this.fetchFiles();
},
},
mounted() {
// We need to wait for `ref` and `projectPath` to be set
this.$nextTick(() => this.fetchFiles());
this.$nextTick(() => {
resetRequestedCommits();
this.fetchFiles();
});
},
methods: {
fetchFiles() {
......@@ -152,6 +158,18 @@ export default {
.concat(data.trees.pageInfo, data.submodules.pageInfo, data.blobs.pageInfo)
.find(({ hasNextPage }) => hasNextPage);
},
loadCommitData({ rowNumber = 0, hasCommit } = {}) {
if (!this.glFeatures.lazyLoadCommits || hasCommit || isRequested(rowNumber)) {
return;
}
loadCommits(this.projectPath, this.path, this.ref, rowNumber)
.then(this.setCommitData)
.catch(() => {});
},
setCommitData(data) {
this.commits = this.commits.concat(data);
},
handleShowMore() {
this.clickedShowMore = true;
this.pagesLoaded += 1;
......@@ -169,7 +187,9 @@ export default {
:is-loading="isLoadingFiles"
:loading-path="loadingPath"
:has-more="hasShowMore"
:commits="commits"
@showMore="handleShowMore"
@row-appear="loadCommitData"
/>
<file-preview v-if="readme" :blob="readme" />
</div>
......
......@@ -4,6 +4,8 @@ export const TREE_PAGE_LIMIT = 1000; // the maximum amount of items per page
export const TREE_PAGE_SIZE = 100; // the amount of items to be fetched per (batch) request
export const TREE_INITIAL_FETCH_COUNT = TREE_PAGE_LIMIT / TREE_PAGE_SIZE; // the amount of (batch) requests to make
export const COMMIT_BATCH_SIZE = 25; // we request commit data in batches of 25
export const SECONDARY_OPTIONS_TEXT = __('Cancel');
export const COMMIT_LABEL = __('Commit message');
export const TARGET_BRANCH_LABEL = __('Target branch');
......@@ -13,3 +15,5 @@ export const COMMIT_MESSAGE_SUBJECT_MAX_LENGTH = 52;
export const COMMIT_MESSAGE_BODY_MAX_LENGTH = 72;
export const LIMITED_CONTAINER_WIDTH_CLASS = 'limit-container-width';
export const I18N_COMMIT_DATA_FETCH_ERROR = __('An error occurred while fetching commit data.');
......@@ -16,6 +16,7 @@ class Projects::TreeController < Projects::ApplicationController
before_action :authorize_edit_tree!, only: [:create_dir]
before_action do
push_frontend_feature_flag(:lazy_load_commits, @project, default_enabled: :yaml)
push_frontend_feature_flag(:paginated_tree_graphql_query, @project, default_enabled: :yaml)
end
......
......@@ -33,6 +33,7 @@ class ProjectsController < Projects::ApplicationController
before_action :export_rate_limit, only: [:export, :download_export, :generate_new_export]
before_action do
push_frontend_feature_flag(:lazy_load_commits, @project, default_enabled: :yaml)
push_frontend_feature_flag(:refactor_blob_viewer, @project, default_enabled: :yaml)
push_frontend_feature_flag(:refactor_text_viewer, @project, default_enabled: :yaml)
push_frontend_feature_flag(:increase_page_size_exponentially, @project, default_enabled: :yaml)
......
---
name: lazy_load_commits
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71633
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/342497
milestone: '14.4'
type: development
group: group::source code
default_enabled: false
......@@ -3611,6 +3611,9 @@ msgstr ""
msgid "An error occurred while fetching codequality mr diff reports."
msgstr ""
msgid "An error occurred while fetching commit data."
msgstr ""
msgid "An error occurred while fetching commits. Retry the search."
msgstr ""
......
import MockAdapter from 'axios-mock-adapter';
import axios from '~/lib/utils/axios_utils';
import { loadCommits, isRequested, resetRequestedCommits } from '~/repository/commits_service';
import httpStatus from '~/lib/utils/http_status';
import createFlash from '~/flash';
import { I18N_COMMIT_DATA_FETCH_ERROR } from '~/repository/constants';
jest.mock('~/flash');
describe('commits service', () => {
let mock;
const url = `${gon.relative_url_root || ''}/my-project/-/refs/main/logs_tree/`;
beforeEach(() => {
mock = new MockAdapter(axios);
mock.onGet(url).reply(httpStatus.OK, [], {});
jest.spyOn(axios, 'get');
});
afterEach(() => {
mock.restore();
resetRequestedCommits();
});
const requestCommits = (offset, project = 'my-project', path = '', ref = 'main') =>
loadCommits(project, path, ref, offset);
it('calls axios get', async () => {
const offset = 10;
const project = 'my-project';
const path = 'my-path';
const ref = 'my-ref';
const testUrl = `${gon.relative_url_root || ''}/${project}/-/refs/${ref}/logs_tree/${path}`;
await requestCommits(offset, project, path, ref);
expect(axios.get).toHaveBeenCalledWith(testUrl, { params: { format: 'json', offset } });
});
it('encodes the path correctly', async () => {
await requestCommits(1, 'some-project', 'with $peci@l ch@rs/');
const encodedUrl = '/some-project/-/refs/main/logs_tree/with%20%24peci%40l%20ch%40rs%2F';
expect(axios.get).toHaveBeenCalledWith(encodedUrl, expect.anything());
});
it('calls axios get once per batch', async () => {
await Promise.all([requestCommits(0), requestCommits(1), requestCommits(23)]);
expect(axios.get.mock.calls.length).toEqual(1);
});
it('calls axios get twice if an offset is larger than 25', async () => {
await requestCommits(100);
expect(axios.get.mock.calls[0][1]).toEqual({ params: { format: 'json', offset: 75 } });
expect(axios.get.mock.calls[1][1]).toEqual({ params: { format: 'json', offset: 100 } });
});
it('updates the list of requested offsets', async () => {
await requestCommits(200);
expect(isRequested(200)).toBe(true);
});
it('resets the list of requested offsets', async () => {
await requestCommits(300);
resetRequestedCommits();
expect(isRequested(300)).toBe(false);
});
it('calls `createFlash` when the request fails', async () => {
const invalidPath = '/#@ some/path';
const invalidUrl = `${url}${invalidPath}`;
mock.onGet(invalidUrl).replyOnce(httpStatus.INTERNAL_SERVER_ERROR, [], {});
await requestCommits(1, 'my-project', invalidPath);
expect(createFlash).toHaveBeenCalledWith({ message: I18N_COMMIT_DATA_FETCH_ERROR });
});
});
......@@ -31,25 +31,36 @@ exports[`Repository table row component renders a symlink table row 1`] = `
<!---->
<!---->
<gl-icon-stub
class="ml-1"
name="lock"
size="12"
title="Locked by Root"
/>
</td>
<td
class="d-none d-sm-table-cell tree-commit cursor-default"
>
<gl-skeleton-loading-stub
class="h-auto"
lines="1"
<gl-link-stub
class="str-truncated-100 tree-commit-link"
/>
<gl-intersection-observer-stub>
<!---->
</gl-intersection-observer-stub>
</td>
<td
class="tree-time-ago text-right cursor-default"
>
<gl-skeleton-loading-stub
class="ml-auto h-auto w-50"
lines="1"
<timeago-tooltip-stub
cssclass=""
time="2019-01-01"
tooltipplacement="top"
/>
<!---->
</td>
</tr>
`;
......@@ -85,25 +96,36 @@ exports[`Repository table row component renders table row 1`] = `
<!---->
<!---->
<gl-icon-stub
class="ml-1"
name="lock"
size="12"
title="Locked by Root"
/>
</td>
<td
class="d-none d-sm-table-cell tree-commit cursor-default"
>
<gl-skeleton-loading-stub
class="h-auto"
lines="1"
<gl-link-stub
class="str-truncated-100 tree-commit-link"
/>
<gl-intersection-observer-stub>
<!---->
</gl-intersection-observer-stub>
</td>
<td
class="tree-time-ago text-right cursor-default"
>
<gl-skeleton-loading-stub
class="ml-auto h-auto w-50"
lines="1"
<timeago-tooltip-stub
cssclass=""
time="2019-01-01"
tooltipplacement="top"
/>
<!---->
</td>
</tr>
`;
......@@ -139,25 +161,36 @@ exports[`Repository table row component renders table row for path with special
<!---->
<!---->
<gl-icon-stub
class="ml-1"
name="lock"
size="12"
title="Locked by Root"
/>
</td>
<td
class="d-none d-sm-table-cell tree-commit cursor-default"
>
<gl-skeleton-loading-stub
class="h-auto"
lines="1"
<gl-link-stub
class="str-truncated-100 tree-commit-link"
/>
<gl-intersection-observer-stub>
<!---->
</gl-intersection-observer-stub>
</td>
<td
class="tree-time-ago text-right cursor-default"
>
<gl-skeleton-loading-stub
class="ml-auto h-auto w-50"
lines="1"
<timeago-tooltip-stub
cssclass=""
time="2019-01-01"
tooltipplacement="top"
/>
<!---->
</td>
</tr>
`;
......@@ -34,17 +34,45 @@ const MOCK_BLOBS = [
},
];
function factory({ path, isLoading = false, hasMore = true, entries = {} }) {
const MOCK_COMMITS = [
{
fileName: 'blob.md',
type: 'blob',
commit: {
message: 'Updated blob.md',
},
},
{
fileName: 'blob2.md',
type: 'blob',
commit: {
message: 'Updated blob2.md',
},
},
{
fileName: 'blob3.md',
type: 'blob',
commit: {
message: 'Updated blob3.md',
},
},
];
function factory({ path, isLoading = false, hasMore = true, entries = {}, commits = [] }) {
vm = shallowMount(Table, {
propsData: {
path,
isLoading,
entries,
hasMore,
commits,
},
mocks: {
$apollo,
},
provide: {
glFeatures: { lazyLoadCommits: true },
},
});
}
......@@ -82,12 +110,15 @@ describe('Repository table component', () => {
entries: {
blobs: MOCK_BLOBS,
},
commits: MOCK_COMMITS,
});
const rows = vm.findAll(TableRow);
expect(rows.length).toEqual(3);
expect(rows.at(2).attributes().mode).toEqual('120000');
expect(rows.at(2).props().rowNumber).toBe(2);
expect(rows.at(2).props().commitInfo).toEqual(MOCK_COMMITS[2]);
});
describe('Show more button', () => {
......
import { GlBadge, GlLink, GlIcon } from '@gitlab/ui';
import { GlBadge, GlLink, GlIcon, GlIntersectionObserver } from '@gitlab/ui';
import { shallowMount, RouterLinkStub } from '@vue/test-utils';
import { createMockDirective, getBinding } from 'helpers/vue_mock_directive';
import TableRow from '~/repository/components/table/row.vue';
import FileIcon from '~/vue_shared/components/file_icon.vue';
import { FILE_SYMLINK_MODE } from '~/vue_shared/constants';
const COMMIT_MOCK = { lockLabel: 'Locked by Root', committedDate: '2019-01-01' };
let vm;
let $router;
......@@ -20,12 +22,14 @@ function factory(propsData = {}) {
projectPath: 'gitlab-org/gitlab-ce',
url: `https://test.com`,
totalEntries: 10,
commitInfo: COMMIT_MOCK,
rowNumber: 123,
},
directives: {
GlHoverLoad: createMockDirective(),
},
provide: {
glFeatures: { refactorBlobViewer: true },
glFeatures: { refactorBlobViewer: true, lazyLoadCommits: true },
},
mocks: {
$router,
......@@ -40,6 +44,7 @@ function factory(propsData = {}) {
describe('Repository table row component', () => {
const findRouterLink = () => vm.find(RouterLinkStub);
const findIntersectionObserver = () => vm.findComponent(GlIntersectionObserver);
afterEach(() => {
vm.destroy();
......@@ -226,8 +231,6 @@ describe('Repository table row component', () => {
currentPath: '/',
});
vm.setData({ commit: { lockLabel: 'Locked by Root', committedDate: '2019-01-01' } });
return vm.vm.$nextTick().then(() => {
expect(vm.find(GlIcon).exists()).toBe(true);
expect(vm.find(GlIcon).props('name')).toBe('lock');
......@@ -246,4 +249,27 @@ describe('Repository table row component', () => {
expect(vm.find(FileIcon).props('loading')).toBe(true);
});
describe('row visibility', () => {
beforeEach(() => {
factory({
id: '1',
sha: '1',
path: 'test',
type: 'tree',
currentPath: '/',
});
});
it('emits a `row-appear` event', () => {
findIntersectionObserver().vm.$emit('appear');
expect(vm.emitted('row-appear')).toEqual([
[
{
hasCommit: true,
rowNumber: 123,
},
],
]);
});
});
});
......@@ -3,6 +3,13 @@ import paginatedTreeQuery from 'shared_queries/repository/paginated_tree.query.g
import FilePreview from '~/repository/components/preview/index.vue';
import FileTable from '~/repository/components/table/index.vue';
import TreeContent from '~/repository/components/tree_content.vue';
import { loadCommits, isRequested, resetRequestedCommits } from '~/repository/commits_service';
jest.mock('~/repository/commits_service', () => ({
loadCommits: jest.fn(() => Promise.resolve()),
isRequested: jest.fn(),
resetRequestedCommits: jest.fn(),
}));
let vm;
let $apollo;
......@@ -23,6 +30,7 @@ function factory(path, data = () => ({})) {
glFeatures: {
increasePageSizeExponentially: true,
paginatedTreeGraphqlQuery: true,
lazyLoadCommits: true,
},
},
});
......@@ -45,7 +53,7 @@ describe('Repository table component', () => {
expect(vm.find(FilePreview).exists()).toBe(true);
});
it('trigger fetchFiles when mounted', async () => {
it('trigger fetchFiles and resetRequestedCommits when mounted', async () => {
factory('/');
jest.spyOn(vm.vm, 'fetchFiles').mockImplementation(() => {});
......@@ -53,6 +61,7 @@ describe('Repository table component', () => {
await vm.vm.$nextTick();
expect(vm.vm.fetchFiles).toHaveBeenCalled();
expect(resetRequestedCommits).toHaveBeenCalled();
});
describe('normalizeData', () => {
......@@ -180,4 +189,15 @@ describe('Repository table component', () => {
});
});
});
it('loads commit data when row-appear event is emitted', () => {
const path = 'some/path';
const rowNumber = 1;
factory(path);
findFileTable().vm.$emit('row-appear', { hasCommit: false, rowNumber });
expect(isRequested).toHaveBeenCalledWith(rowNumber);
expect(loadCommits).toHaveBeenCalledWith('', path, '', rowNumber);
});
});
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment