Skip to content

G
gitlab-ce
Commit aa5831c2 authored by Marcia Ramos's avatar Marcia Ramos
Browse files
Options

Merge branch 'docs-move-lfs-doc' into 'master' 

Move lfs user doc to git topics directory

Closes #210354

See merge request gitlab-org/gitlab!27191
parents 2e9b3a73 5e7f259d
Expand all Show whitespace changes
Inline Side-by-side
Showing with 735 additions and 721 deletions
app/helpers/projects_helper.rb
View file @ aa5831c2
...@@ -602,9 +602,9 @@ module ProjectsHelper ...@@ -602,9 +602,9 @@ module ProjectsHelper
registryAvailable: Gitlab.config.registry.enabled, registryAvailable: Gitlab.config.registry.enabled,
registryHelpPath: help_page_path('user/packages/container_registry/index'), registryHelpPath: help_page_path('user/packages/container_registry/index'),
lfsAvailable: Gitlab.config.lfs.enabled, lfsAvailable: Gitlab.config.lfs.enabled,
lfsHelpPath: help_page_path('workflow/lfs/manage_large_binaries_with_git_lfs'), lfsHelpPath: help_page_path('topics/git/lfs/index'),
lfsObjectsExist: project.lfs_objects.exists?, lfsObjectsExist: project.lfs_objects.exists?,
lfsObjectsRemovalHelpPath: help_page_path('administration/lfs/manage_large_binaries_with_git_lfs', anchor: 'removing-objects-from-lfs'), lfsObjectsRemovalHelpPath: help_page_path('topics/git/lfs/index', anchor: 'removing-objects-from-lfs'),
pagesAvailable: Gitlab.config.pages.enabled, pagesAvailable: Gitlab.config.pages.enabled,
pagesAccessControlEnabled: Gitlab.config.pages.access_control, pagesAccessControlEnabled: Gitlab.config.pages.access_control,
pagesAccessControlForced: ::Gitlab::Pages.access_control_is_forced?, pagesAccessControlForced: ::Gitlab::Pages.access_control_is_forced?,
......
app/views/admin/groups/show.html.haml
View file @ aa5831c2
...@@ -55,7 +55,7 @@ ...@@ -55,7 +55,7 @@
%span.light= _('Group Git LFS status:') %span.light= _('Group Git LFS status:')
%strong %strong
= group_lfs_status(@group) = group_lfs_status(@group)
= link_to icon('question-circle'), help_page_path('workflow/lfs/manage_large_binaries_with_git_lfs') = link_to icon('question-circle'), help_page_path('topics/git/lfs/index')
= render_if_exists 'namespaces/shared_runner_status', namespace: @group = render_if_exists 'namespaces/shared_runner_status', namespace: @group
......
app/views/admin/projects/show.html.haml
View file @ aa5831c2
...@@ -89,7 +89,7 @@ ...@@ -89,7 +89,7 @@
%span.light Git LFS status: %span.light Git LFS status:
%strong %strong
= project_lfs_status(@project) = project_lfs_status(@project)
= link_to icon('question-circle'), help_page_path('workflow/lfs/manage_large_binaries_with_git_lfs') = link_to icon('question-circle'), help_page_path('topics/git/lfs/index')
- else - else
%li %li
%span.light repository: %span.light repository:
......
app/views/groups/_group_admin_settings.html.haml
View file @ aa5831c2
...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
= f.label :lfs_enabled, class: 'form-check-label' do = f.label :lfs_enabled, class: 'form-check-label' do
%strong %strong
Allow projects within this group to use Git LFS Allow projects within this group to use Git LFS
= link_to icon('question-circle'), help_page_path('workflow/lfs/manage_large_binaries_with_git_lfs') = link_to icon('question-circle'), help_page_path('topics/git/lfs/index')
%br/ %br/
%span This setting can be overridden in each project. %span This setting can be overridden in each project.
.form-group.row .form-group.row
......
app/views/groups/settings/_lfs.html.haml
View file @ aa5831c2
- docs_link_url = help_page_path('workflow/lfs/manage_large_binaries_with_git_lfs') - docs_link_url = help_page_path('topics/git/lfs/index')
- docs_link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: docs_link_url } - docs_link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: docs_link_url }
%h5= _('Large File Storage') %h5= _('Large File Storage')
......
doc/administration/git_annex.md
View file @ aa5831c2
...@@ -236,7 +236,7 @@ push origin ...@@ -236,7 +236,7 @@ push origin
[deprecate-annex-issue]: https://gitlab.com/gitlab-org/gitlab/issues/1648 [deprecate-annex-issue]: https://gitlab.com/gitlab-org/gitlab/issues/1648
[git-annex]: https://git-annex.branchable.com/ "git-annex website" [git-annex]: https://git-annex.branchable.com/ "git-annex website"
[gitlab shell]: https://gitlab.com/gitlab-org/gitlab-shell "GitLab Shell repository" [gitlab shell]: https://gitlab.com/gitlab-org/gitlab-shell "GitLab Shell repository"
[guide]: lfs/migrate_from_git_annex_to_git_lfs.html [guide]: ../topics/git/lfs/migrate_from_git_annex_to_git_lfs.md
[issue]: https://git-annex.branchable.com/forum/Error_from_git-annex-shell_on_creation_of_gcrypt_special_remote/ "git-annex issue" [issue]: https://git-annex.branchable.com/forum/Error_from_git-annex-shell_on_creation_of_gcrypt_special_remote/ "git-annex issue"
[reconfigure GitLab]: restart_gitlab.md#omnibus-gitlab-reconfigure [reconfigure GitLab]: restart_gitlab.md#omnibus-gitlab-reconfigure
[restart GitLab]: restart_gitlab.md#installations-from-source [restart GitLab]: restart_gitlab.md#installations-from-source
doc/administration/lfs/lfs_administration.md
View file @ aa5831c2
...@@ -4,7 +4,7 @@ disqus_identifier: 'https://docs.gitlab.com/ee/workflow/lfs/lfs_administration.h ...@@ -4,7 +4,7 @@ disqus_identifier: 'https://docs.gitlab.com/ee/workflow/lfs/lfs_administration.h
# GitLab Git LFS Administration # GitLab Git LFS Administration
Documentation on how to use Git LFS are under [Managing large binary files with Git LFS doc](manage_large_binaries_with_git_lfs.md). Documentation on how to use Git LFS are under [Managing large binary files with Git LFS doc](../../topics/git/lfs/index.md).
## Requirements ## Requirements
......
doc/administration/lfs/migrate_from_git_annex_to_git_lfs.md
View file @ aa5831c2
# Migration guide from Git Annex to Git LFS ---
redirect_to: '../../topics/git/lfs/migrate_from_git_annex_to_git_lfs.md'
>**Note:**
Git Annex support [has been removed][issue-remove-annex] in GitLab Enterprise
Edition 9.0 (2017/03/22).
Both [Git Annex][] and [Git LFS][] are tools to manage large files in Git.
## History
Git Annex [was introduced in GitLab Enterprise Edition 7.8][post-3], at a time
where Git LFS didn't yet exist. A few months later, GitLab brought support for
Git LFS in [GitLab 8.2][post-2] and is available for both Community and
Enterprise editions.
## Differences between Git Annex and Git LFS
Some items below are general differences between the two protocols and some are
ones that GitLab developed.
- Git Annex works only through SSH, whereas Git LFS works both with SSH and HTTPS
(SSH support was added in GitLab 8.12).
- Annex files are stored in a sub-directory of the normal repositories, whereas
LFS files are stored outside of the repositories in a place you can define.
- Git Annex requires a more complex setup, but has much more options than Git
LFS. You can compare the commands each one offers by running `man git-annex`
and `man git-lfs`.
- Annex files cannot be browsed directly in GitLab's interface, whereas LFS
files can.
## Migration steps
>**Note:**
Since Git Annex files are stored in a sub-directory of the normal repositories
(`.git/annex/objects`) and LFS files are stored outside of the repositories,
they are not compatible as they are using a different scheme. Therefore, the
migration has to be done manually per repository.
There are basically two steps you need to take in order to migrate from Git
Annex to Git LFS.
### TL; DR
If you know what you are doing and want to skip the reading, this is what you
need to do (we assume you have [git-annex enabled](../git_annex.md#using-gitlab-git-annex) in your
repository and that you have made backups in case something goes wrong).
Fire up a terminal, navigate to your Git repository and:
1. Disable `git-annex`:
```shell
git annex sync --content
git annex direct
git annex uninit
git annex indirect
```
1. Enable `git-lfs`:
```shell
git lfs install
git lfs track <files>
git add .
git commit -m "commit message"
git push
```
### Disabling Git Annex in your repo
Before changing anything, make sure you have a backup of your repository first.
There are a couple of ways to do that, but you can simply clone it to another
local path and maybe push it to GitLab if you want a remote backup as well.
Here you'll find a guide on
[how to back up a **git-annex** repository to an external hard drive][bkp-ext-drive].
Since Annex files are stored as objects with symlinks and cannot be directly
modified, we need to first remove those symlinks.
NOTE: **Note:**
Make sure the you read about the [`direct` mode][annex-direct] as it contains
useful information that may fit in your use case. Note that `annex direct` is
deprecated in Git Annex version 6, so you may need to upgrade your repository
if the server also has Git Annex 6 installed. Read more in the
[Git Annex troubleshooting tips](../git_annex.md#troubleshooting-tips) section.
1. Backup your repository
```shell
cd repository
git annex sync --content
cd ..
git clone repository repository-backup
cd repository-backup
git annex get
cd ..
```
1. Use `annex direct`:
```shell
cd repository
git annex direct
```
The output should be similar to this:
```shell
commit
On branch master
Your branch is up-to-date with 'origin/master'.
nothing to commit, working tree clean
ok
direct debian.iso ok
direct ok
```
1. Disable Git Annex with [`annex uninit`][uninit]:
```shell
git annex uninit
```
The output should be similar to this:
```shell
unannex debian.iso ok
Deleted branch git-annex (was 2534d2c).
```
This will `unannex` every file in the repository, leaving the original files.
1. Switch back to `indirect` mode:
```shell
git annex indirect
```
The output should be similar to this:
```shell
(merging origin/git-annex into git-annex...)
(recording state in git...)
commit (recording state in git...)
ok
(recording state in git...)
[master fac3194] commit before switching to indirect mode
1 file changed, 1 deletion(-)
delete mode 120000 alpine-virt-3.4.4-x86_64.iso
ok
indirect ok
ok
```
--- ---
At this point, you have two options. Either add, commit and push the files This document was moved to [another location](../../topics/git/lfs/migrate_from_git_annex_to_git_lfs.md).
directly back to GitLab or switch to Git LFS. We will tackle the LFS switch in
the next section.
### Enabling Git LFS in your repo
Git LFS is enabled by default on all GitLab products (GitLab CE, GitLab EE,
GitLab.com), therefore, you don't need to do anything server-side.
1. First, make sure you have `git-lfs` installed locally:
```shell
git lfs help
```
If the terminal doesn't prompt you with a full response on `git-lfs` commands,
[install the Git LFS client][install-lfs] first.
1. Inside the repo, run the following command to initiate LFS:
```shell
git lfs install
```
1. Enable `git-lfs` for the group of files you want to track. You
can track specific files, all files containing the same extension, or an
entire directory:
```shell
git lfs track images/01.png # per file
git lfs track **/*.png # per extension
git lfs track images/ # per directory
```
Once you do that, run `git status` and you'll see `.gitattributes` added
to your repo. It collects all file patterns that you chose to track via
`git-lfs`.
1. Add the files, commit and push them to GitLab:
```shell
git add .
git commit -m "commit message"
git push
```
If your remote is set up with HTTP, you will be asked to enter your login
credentials. If you have [2FA enabled](../../user/profile/account/two_factor_authentication.md), make sure to use a
[personal access token](../../user/profile/account/two_factor_authentication.md#personal-access-tokens)
instead of your password.
## Removing the Git Annex branches
After the migration finishes successfully, you can remove all `git-annex`
related branches from your repository.
On GitLab, navigate to your project's **Repository ➔ Branches** and delete all
branches created by Git Annex: `git-annex`, and all under `synced/`.
![repository branches](img/git-annex-branches.png)
You can also do this on the command line with:
```shell
git branch -d synced/master
git branch -d synced/git-annex
git push origin :synced/master
git push origin :synced/git-annex
git push origin :git-annex
git remote prune origin
```
If there are still some Annex objects inside your repository (`.git/annex/`)
or references inside `.git/config`, run `annex uninit` again:
```shell
git annex uninit
```
## Further Reading
- (Blog Post) [Getting Started with Git FLS][post-1]
- (Blog Post) [Announcing LFS Support in GitLab][post-2]
- (Blog Post) [GitLab Annex Solves the Problem of Versioning Large Binaries with Git][post-3]
- (GitLab Docs) [Git Annex](../git_annex.md)
- (GitLab Docs) [Git LFS](manage_large_binaries_with_git_lfs.md)
[annex-direct]: https://git-annex.branchable.com/direct_mode/
[bkp-ext-drive]: https://www.thomas-krenn.com/en/wiki/Git-annex_Repository_on_an_External_Hard_Drive
[Git Annex]: http://git-annex.branchable.com/
[Git LFS]: https://git-lfs.github.com/
[install-lfs]: https://git-lfs.github.com/
[issue-remove-annex]: https://gitlab.com/gitlab-org/gitlab/issues/1648
[lfs-track]: https://about.gitlab.com/blog/2017/01/30/getting-started-with-git-lfs-tutorial/#tracking-files-with-lfs
[post-1]: https://about.gitlab.com/blog/2017/01/30/getting-started-with-git-lfs-tutorial/
[post-2]: https://about.gitlab.com/blog/2015/11/23/announcing-git-lfs-support-in-gitlab/
[post-3]: https://about.gitlab.com/blog/2015/02/17/gitlab-annex-solves-the-problem-of-versioning-large-binaries-with-git/
[uninit]: https://git-annex.branchable.com/git-annex-uninit/
doc/administration/repository_storage_types.md
View file @ aa5831c2
...@@ -235,7 +235,7 @@ CI Artifacts are S3 compatible since **9.4** (GitLab Premium), and available in ...@@ -235,7 +235,7 @@ CI Artifacts are S3 compatible since **9.4** (GitLab Premium), and available in
##### LFS Objects ##### LFS Objects
[LFS Objects in GitLab](lfs/manage_large_binaries_with_git_lfs.md) implement a similar [LFS Objects in GitLab](../topics/git/lfs/index.md) implement a similar
storage pattern using 2 chars, 2 level folders, following Git's own implementation: storage pattern using 2 chars, 2 level folders, following Git's own implementation:
```ruby ```ruby
......
doc/development/lfs.md
View file @ aa5831c2
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
In April 2019, Francisco Javier López hosted a [Deep Dive] on GitLab's [Git LFS] implementation to share his domain specific knowledge with anyone who may work in this part of the code base in the future. You can find the [recording on YouTube], and the slides on [Google Slides] and in [PDF]. Everything covered in this deep dive was accurate as of GitLab 11.10, and while specific details may have changed since then, it should still serve as a good introduction. In April 2019, Francisco Javier López hosted a [Deep Dive] on GitLab's [Git LFS] implementation to share his domain specific knowledge with anyone who may work in this part of the code base in the future. You can find the [recording on YouTube], and the slides on [Google Slides] and in [PDF]. Everything covered in this deep dive was accurate as of GitLab 11.10, and while specific details may have changed since then, it should still serve as a good introduction.
[Deep Dive]: https://gitlab.com/gitlab-org/create-stage/issues/1 [Deep Dive]: https://gitlab.com/gitlab-org/create-stage/issues/1
[Git LFS]: ../administration/lfs/manage_large_binaries_with_git_lfs.md [Git LFS]: ../topics/git/lfs/index.md
[recording on YouTube]: https://www.youtube.com/watch?v=Yyxwcksr0Qc [recording on YouTube]: https://www.youtube.com/watch?v=Yyxwcksr0Qc
[Google Slides]: https://docs.google.com/presentation/d/1E-aw6-z0rYd0346YhIWE7E9A65zISL9iIMAOq2zaw9E/edit [Google Slides]: https://docs.google.com/presentation/d/1E-aw6-z0rYd0346YhIWE7E9A65zISL9iIMAOq2zaw9E/edit
[PDF]: https://gitlab.com/gitlab-org/create-stage/uploads/07a89257a140db067bdfb484aecd35e1/Git_LFS_Deep_Dive__Create_.pdf [PDF]: https://gitlab.com/gitlab-org/create-stage/uploads/07a89257a140db067bdfb484aecd35e1/Git_LFS_Deep_Dive__Create_.pdf
doc/topics/git/index.md
View file @ aa5831c2
...@@ -84,9 +84,9 @@ Git-related queries from GitLab. ...@@ -84,9 +84,9 @@ Git-related queries from GitLab.
The following relate to Git Large File Storage: The following relate to Git Large File Storage:
- [Getting Started with Git LFS](https://about.gitlab.com/blog/2017/01/30/getting-started-with-git-lfs-tutorial/) - [Getting Started with Git LFS](https://about.gitlab.com/blog/2017/01/30/getting-started-with-git-lfs-tutorial/)
- [Migrate an existing Git repo with Git LFS](migrate_to_git_lfs/index.md) - [Migrate an existing Git repo with Git LFS](lfs/migrate_to_git_lfs.md)
- [Removing objects from LFS](../../administration/lfs/manage_large_binaries_with_git_lfs.md#removing-objects-from-lfs) - [Removing objects from LFS](lfs/index.md#removing-objects-from-lfs)
- [GitLab Git LFS user documentation](../../administration/lfs/manage_large_binaries_with_git_lfs.md) - [GitLab Git LFS user documentation](lfs/index.md)
- [GitLab Git LFS admin documentation](../../administration/lfs/lfs_administration.md) - [GitLab Git LFS admin documentation](../../administration/lfs/lfs_administration.md)
- [git-annex to Git-LFS migration guide](../../administration/lfs/migrate_from_git_annex_to_git_lfs.md) - [git-annex to Git-LFS migration guide](lfs/migrate_from_git_annex_to_git_lfs.md)
- [Towards a production quality open source Git LFS server](https://about.gitlab.com/blog/2015/08/13/towards-a-production-quality-open-source-git-lfs-server/) - [Towards a production quality open source Git LFS server](https://about.gitlab.com/blog/2015/08/13/towards-a-production-quality-open-source-git-lfs-server/)
doc/topics/git/lfs/index.md 0 → 100644
View file @ aa5831c2
This diff is collapsed.
doc/topics/git/lfs/migrate_from_git_annex_to_git_lfs.md 0 → 100644
View file @ aa5831c2
# Migration guide from Git Annex to Git LFS
>**Note:**
Git Annex support [has been removed][issue-remove-annex] in GitLab Enterprise
Edition 9.0 (2017/03/22).
Both [Git Annex][] and [Git LFS][] are tools to manage large files in Git.
## History
Git Annex [was introduced in GitLab Enterprise Edition 7.8][post-3], at a time
where Git LFS didn't yet exist. A few months later, GitLab brought support for
Git LFS in [GitLab 8.2][post-2] and is available for both Community and
Enterprise editions.
## Differences between Git Annex and Git LFS
Some items below are general differences between the two protocols and some are
ones that GitLab developed.
- Git Annex works only through SSH, whereas Git LFS works both with SSH and HTTPS
(SSH support was added in GitLab 8.12).
- Annex files are stored in a sub-directory of the normal repositories, whereas
LFS files are stored outside of the repositories in a place you can define.
- Git Annex requires a more complex setup, but has much more options than Git
LFS. You can compare the commands each one offers by running `man git-annex`
and `man git-lfs`.
- Annex files cannot be browsed directly in GitLab's interface, whereas LFS
files can.
## Migration steps
>**Note:**
Since Git Annex files are stored in a sub-directory of the normal repositories
(`.git/annex/objects`) and LFS files are stored outside of the repositories,
they are not compatible as they are using a different scheme. Therefore, the
migration has to be done manually per repository.
There are basically two steps you need to take in order to migrate from Git
Annex to Git LFS.
### TL; DR
If you know what you are doing and want to skip the reading, this is what you
need to do (we assume you have [git-annex enabled](../../../administration/git_annex.md#using-gitlab-git-annex) in your
repository and that you have made backups in case something goes wrong).
Fire up a terminal, navigate to your Git repository and:
1. Disable `git-annex`:
```shell
git annex sync --content
git annex direct
git annex uninit
git annex indirect
```
1. Enable `git-lfs`:
```shell
git lfs install
git lfs track <files>
git add .
git commit -m "commit message"
git push
```
### Disabling Git Annex in your repo
Before changing anything, make sure you have a backup of your repository first.
There are a couple of ways to do that, but you can simply clone it to another
local path and maybe push it to GitLab if you want a remote backup as well.
Here you'll find a guide on
[how to back up a **git-annex** repository to an external hard drive][bkp-ext-drive].
Since Annex files are stored as objects with symlinks and cannot be directly
modified, we need to first remove those symlinks.
NOTE: **Note:**
Make sure the you read about the [`direct` mode][annex-direct] as it contains
useful information that may fit in your use case. Note that `annex direct` is
deprecated in Git Annex version 6, so you may need to upgrade your repository
if the server also has Git Annex 6 installed. Read more in the
[Git Annex troubleshooting tips](../../../administration/git_annex.md#troubleshooting-tips) section.
1. Backup your repository
```shell
cd repository
git annex sync --content
cd ..
git clone repository repository-backup
cd repository-backup
git annex get
cd ..
```
1. Use `annex direct`:
```shell
cd repository
git annex direct
```
The output should be similar to this:
```shell
commit
On branch master
Your branch is up-to-date with 'origin/master'.
nothing to commit, working tree clean
ok
direct debian.iso ok
direct ok
```
1. Disable Git Annex with [`annex uninit`][uninit]:
```shell
git annex uninit
```
The output should be similar to this:
```shell
unannex debian.iso ok
Deleted branch git-annex (was 2534d2c).
```
This will `unannex` every file in the repository, leaving the original files.
1. Switch back to `indirect` mode:
```shell
git annex indirect
```
The output should be similar to this:
```shell
(merging origin/git-annex into git-annex...)
(recording state in git...)
commit (recording state in git...)
ok
(recording state in git...)
[master fac3194] commit before switching to indirect mode
1 file changed, 1 deletion(-)
delete mode 120000 alpine-virt-3.4.4-x86_64.iso
ok
indirect ok
ok
```
---
At this point, you have two options. Either add, commit and push the files
directly back to GitLab or switch to Git LFS. We will tackle the LFS switch in
the next section.
### Enabling Git LFS in your repo
Git LFS is enabled by default on all GitLab products (GitLab CE, GitLab EE,
GitLab.com), therefore, you don't need to do anything server-side.
1. First, make sure you have `git-lfs` installed locally:
```shell
git lfs help
```
If the terminal doesn't prompt you with a full response on `git-lfs` commands,
[install the Git LFS client][install-lfs] first.
1. Inside the repo, run the following command to initiate LFS:
```shell
git lfs install
```
1. Enable `git-lfs` for the group of files you want to track. You
can track specific files, all files containing the same extension, or an
entire directory:
```shell
git lfs track images/01.png # per file
git lfs track **/*.png # per extension
git lfs track images/ # per directory
```
Once you do that, run `git status` and you'll see `.gitattributes` added
to your repo. It collects all file patterns that you chose to track via
`git-lfs`.
1. Add the files, commit and push them to GitLab:
```shell
git add .
git commit -m "commit message"
git push
```
If your remote is set up with HTTP, you will be asked to enter your login
credentials. If you have [2FA enabled](../../../user/profile/account/two_factor_authentication.md), make sure to use a
[personal access token](../../../user/profile/account/two_factor_authentication.md#personal-access-tokens)
instead of your password.
## Removing the Git Annex branches
After the migration finishes successfully, you can remove all `git-annex`
related branches from your repository.
On GitLab, navigate to your project's **Repository ➔ Branches** and delete all
branches created by Git Annex: `git-annex`, and all under `synced/`.
![repository branches](img/git-annex-branches.png)
You can also do this on the command line with:
```shell
git branch -d synced/master
git branch -d synced/git-annex
git push origin :synced/master
git push origin :synced/git-annex
git push origin :git-annex
git remote prune origin
```
If there are still some Annex objects inside your repository (`.git/annex/`)
or references inside `.git/config`, run `annex uninit` again:
```shell
git annex uninit
```
## Further Reading
- (Blog Post) [Getting Started with Git FLS][post-1]
- (Blog Post) [Announcing LFS Support in GitLab][post-2]
- (Blog Post) [GitLab Annex Solves the Problem of Versioning Large Binaries with Git][post-3]
- (GitLab Docs) [Git Annex](../../../administration/git_annex.md)
- (GitLab Docs) [Git LFS](index.md)
[annex-direct]: https://git-annex.branchable.com/direct_mode/
[bkp-ext-drive]: https://www.thomas-krenn.com/en/wiki/Git-annex_Repository_on_an_External_Hard_Drive
[Git Annex]: http://git-annex.branchable.com/
[Git LFS]: https://git-lfs.github.com/
[install-lfs]: https://git-lfs.github.com/
[issue-remove-annex]: https://gitlab.com/gitlab-org/gitlab/issues/1648
[post-1]: https://about.gitlab.com/blog/2017/01/30/getting-started-with-git-lfs-tutorial/
[post-2]: https://about.gitlab.com/blog/2015/11/23/announcing-git-lfs-support-in-gitlab/
[post-3]: https://about.gitlab.com/blog/2015/02/17/gitlab-annex-solves-the-problem-of-versioning-large-binaries-with-git/
[uninit]: https://git-annex.branchable.com/git-annex-uninit/
doc/topics/git/lfs/migrate_to_git_lfs.md 0 → 100644
View file @ aa5831c2
---
type: tutorial, concepts
description: "How to migrate an existing Git repository to Git LFS with BFG."
last_updated: 2019-07-11
---
# Migrate a Git repo into Git LFS with BFG
Using Git LFS can help you to reduce the size of your Git
repository and improve its performance.
However, simply adding the
large files that are already in your repository to Git LFS,
will not actually reduce the size of your repository because
the files are still referenced by previous commits.
Through the method described on this document, first migrate
to Git LFS with a tool such as the open source community-maintained [BFG](https://rtyley.github.io/bfg-repo-cleaner/)
through a mirror repo, then clean up the repository's history,
and lastly create LFS tracking rules to prevent new binary files
from being added.
This tutorial was inspired by the guide
[Use BFG to migrate a repo to Git LFS](https://confluence.atlassian.com/bitbucket/use-bfg-to-migrate-a-repo-to-git-lfs-834233484.html).
For more information on Git LFS, see the [references](#references)
below.
CAUTION: **Warning:**
The method described on this guide rewrites Git history. Make
sure to back up your repo before beginning and use it at your
own risk.
## Requirements
Before beginning, make sure:
- You have enough LFS storage for the files you want to convert.
Storage is required for the entire history of all files.
- All the team members you share the repository with have pushed all changes.
Branches based on the repository before applying this method cannot be merged.
Branches based on the repo before applying this method cannot be merged.
To follow this tutorial, you'll need:
- Maintainer permissions to the existing Git repository
you'd like to migrate to LFS with access through the command line.
- [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
and [Java Runtime Environment](https://www.java.com/en/download/manual.jsp)
(Java 7 or above) installed locally.
- BFG installed locally:
```shell
brew install bfg
```
- Git LFS installed locally:
```shell
brew install git-lfs
```
NOTE: **Note:**
This guide was tested on macOS Mojave.
## Steps
Consider an example upstream project, `git@gitlab.com:gitlab-tests/test-git-lfs-repo-migration.git`.
1. Back up your repository:
Create a copy of your repository so that you can
recover it in case something goes wrong.
1. Clone `--mirror` the repo:
Cloning with the mirror flag will create a bare repository.
This ensures you get all the branches within the repo.
It creates a directory called `<repo-name>.git`
(in our example, `test-git-lfs-repo-migration.git`),
mirroring the upstream project:
```shell
git clone --mirror git@gitlab.com:gitlab-tests/test-git-lfs-repo-migration.git
```
1. Convert the Git history with BFG:
```shell
bfg --convert-to-git-lfs "*.{png,mp4,jpg,gif}" --no-blob-protection test-git-lfs-repo-migration.git
```
It is scanning all the history, and looking for any files with
that extension, and then converting them to an LFS pointer.
1. Clean up the repository:
```shell
# cd path/to/mirror/repo:
cd test-git-lfs-repo-migration.git
# clean up the repo:
git reflog expire --expire=now --all && git gc --prune=now --aggressive
```
You can also take a look on how to further [clean the repo](../../../user/project/repository/reducing_the_repo_size_using_git.md),
but it's not necessary for the purposes of this guide.
1. Install Git LFS in the mirror repository:
```shell
git lfs install
```
1. [Unprotect the default branch](../../../user/project/protected_branches.md),
so that we can force-push the rewritten repository:
1. Navigate to your project's **Settings > Repository** and
expand **Protected Branches**.
1. Scroll down to locate the protected branches and click
**Unprotect** the default branch.
1. Force-push to GitLab:
```shell
git push --force
```
1. Track the files you want with LFS:
```shell
# cd path/to/upstream/repo:
cd test-git-lfs-repo-migration
# You may need to reset your local copy with upstream's `master` after force-pushing from the mirror:
git reset --hard origin/master
# Track the files with LFS:
git lfs track "*.gif" "*.png" "*.jpg" "*.psd" "*.mp4" "img/"
```
Now all existing the files you converted, as well as the new
ones you add, will be properly tracked with LFS.
1. [Re-protect the default branch](../../../user/project/protected_branches.md):
1. Navigate to your project's **Settings > Repository** and
expand **Protected Branches**.
1. Select the default branch from the **Branch** dropdown menu,
and set up the
**Allowed to push** and **Allowed to merge** rules.
1. Click **Protect**.
<!-- ## Troubleshooting
Include any troubleshooting steps that you can foresee. If you know beforehand what issues
one might have when setting this up, or when something is changed, or on upgrading, it's
important to describe those, too. Think of things that may go wrong and include them here.
This is important to minimize requests for support, and to avoid doc comments with
questions that you know someone might ask.
Each scenario can be a third-level heading, e.g. `### Getting error message X`.
If you have none to add when creating a doc, leave this section in place
but commented out to help encourage others to add to it in the future. -->
## References
- [Getting Started with Git LFS](https://about.gitlab.com/blog/2017/01/30/getting-started-with-git-lfs-tutorial/)
- [Migrate from Git Annex to Git LFS](migrate_from_git_annex_to_git_lfs.md)
- [GitLab's Git LFS user documentation](index.md)
- [GitLab's Git LFS administrator documentation](../../../administration/lfs/lfs_administration.md)
- Alternative method to [migrate an existing repo to Git LFS](https://github.com/git-lfs/git-lfs/wiki/Tutorial#migrating-existing-repository-data-to-lfs)
<!--
Test project:
https://gitlab.com/gitlab-tests/test-git-lfs-repo-migration
-->
doc/topics/git/migrate_to_git_lfs/index.md
View file @ aa5831c2
--- ---
type: tutorial, concepts redirect_to: '../lfs/migrate_to_git_lfs.md'
description: "How to migrate an existing Git repository to Git LFS with BFG."
last_updated: 2019-07-11
--- ---
# Migrate a Git repo into Git LFS with BFG This document was moved to [another location](../lfs/migrate_to_git_lfs.md).
Using Git LFS can help you to reduce the size of your Git
repository and improve its performance.
However, simply adding the
large files that are already in your repository to Git LFS,
will not actually reduce the size of your repository because
the files are still referenced by previous commits.
Through the method described on this document, first migrate
to Git LFS with a tool such as the open source community-maintained [BFG](https://rtyley.github.io/bfg-repo-cleaner/)
through a mirror repo, then clean up the repository's history,
and lastly create LFS tracking rules to prevent new binary files
from being added.
This tutorial was inspired by the guide
[Use BFG to migrate a repo to Git LFS](https://confluence.atlassian.com/bitbucket/use-bfg-to-migrate-a-repo-to-git-lfs-834233484.html).
For more information on Git LFS, see the [references](#references)
below.
CAUTION: **Warning:**
The method described on this guide rewrites Git history. Make
sure to back up your repo before beginning and use it at your
own risk.
## Requirements
Before beginning, make sure:
- You have enough LFS storage for the files you want to convert.
Storage is required for the entire history of all files.
- All the team members you share the repository with have pushed all changes.
Branches based on the repository before applying this method cannot be merged.
Branches based on the repo before applying this method cannot be merged.
To follow this tutorial, you'll need:
- Maintainer permissions to the existing Git repository
you'd like to migrate to LFS with access through the command line.
- [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
and [Java Runtime Environment](https://www.java.com/en/download/manual.jsp)
(Java 7 or above) installed locally.
- BFG installed locally:
```shell
brew install bfg
```
- Git LFS installed locally:
```shell
brew install git-lfs
```
NOTE: **Note:**
This guide was tested on macOS Mojave.
## Steps
Consider an example upstream project, `git@gitlab.com:gitlab-tests/test-git-lfs-repo-migration.git`.
1. Back up your repository:
Create a copy of your repository so that you can
recover it in case something goes wrong.
1. Clone `--mirror` the repo:
Cloning with the mirror flag will create a bare repository.
This ensures you get all the branches within the repo.
It creates a directory called `<repo-name>.git`
(in our example, `test-git-lfs-repo-migration.git`),
mirroring the upstream project:
```shell
git clone --mirror git@gitlab.com:gitlab-tests/test-git-lfs-repo-migration.git
```
1. Convert the Git history with BFG:
```shell
bfg --convert-to-git-lfs "*.{png,mp4,jpg,gif}" --no-blob-protection test-git-lfs-repo-migration.git
```
It is scanning all the history, and looking for any files with
that extension, and then converting them to an LFS pointer.
1. Clean up the repository:
```shell
# cd path/to/mirror/repo:
cd test-git-lfs-repo-migration.git
# clean up the repo:
git reflog expire --expire=now --all && git gc --prune=now --aggressive
```
You can also take a look on how to further [clean the repo](../../../user/project/repository/reducing_the_repo_size_using_git.md),
but it's not necessary for the purposes of this guide.
1. Install Git LFS in the mirror repository:
```shell
git lfs install
```
1. [Unprotect the default branch](../../../user/project/protected_branches.md),
so that we can force-push the rewritten repository:
1. Navigate to your project's **Settings > Repository** and
expand **Protected Branches**.
1. Scroll down to locate the protected branches and click
**Unprotect** the default branch.
1. Force-push to GitLab:
```shell
git push --force
```
1. Track the files you want with LFS:
```shell
# cd path/to/upstream/repo:
cd test-git-lfs-repo-migration
# You may need to reset your local copy with upstream's `master` after force-pushing from the mirror:
git reset --hard origin/master
# Track the files with LFS:
git lfs track "*.gif" "*.png" "*.jpg" "*.psd" "*.mp4" "img/"
```
Now all existing the files you converted, as well as the new
ones you add, will be properly tracked with LFS.
1. [Re-protect the default branch](../../../user/project/protected_branches.md):
1. Navigate to your project's **Settings > Repository** and
expand **Protected Branches**.
1. Select the default branch from the **Branch** dropdown menu,
and set up the
**Allowed to push** and **Allowed to merge** rules.
1. Click **Protect**.
<!-- ## Troubleshooting
Include any troubleshooting steps that you can foresee. If you know beforehand what issues
one might have when setting this up, or when something is changed, or on upgrading, it's
important to describe those, too. Think of things that may go wrong and include them here.
This is important to minimize requests for support, and to avoid doc comments with
questions that you know someone might ask.
Each scenario can be a third-level heading, e.g. `### Getting error message X`.
If you have none to add when creating a doc, leave this section in place
but commented out to help encourage others to add to it in the future. -->
## References
- [Getting Started with Git LFS](https://about.gitlab.com/blog/2017/01/30/getting-started-with-git-lfs-tutorial/)
- [Migrate from Git Annex to Git LFS](../../../administration/lfs/migrate_from_git_annex_to_git_lfs.md)
- [GitLab's Git LFS user documentation](../../../administration/lfs/manage_large_binaries_with_git_lfs.md)
- [GitLab's Git LFS administrator documentation](../../../administration/lfs/lfs_administration.md)
- Alternative method to [migrate an existing repo to Git LFS](https://github.com/git-lfs/git-lfs/wiki/Tutorial#migrating-existing-repository-data-to-lfs)
<!--
Test project:
https://gitlab.com/gitlab-tests/test-git-lfs-repo-migration
-->
doc/university/support/README.md
View file @ aa5831c2
...@@ -170,7 +170,7 @@ Some tickets need specific knowledge or a deep understanding of a particular com ...@@ -170,7 +170,7 @@ Some tickets need specific knowledge or a deep understanding of a particular com
Move on to understanding some of GitLab's more advanced features. You can make use of GitLab.com to understand the features from an end-user perspective and then use your own instance to understand setup and configuration of the feature from an Administrative perspective Move on to understanding some of GitLab's more advanced features. You can make use of GitLab.com to understand the features from an end-user perspective and then use your own instance to understand setup and configuration of the feature from an Administrative perspective
- Set up and try [Git LFS](../../administration/lfs/manage_large_binaries_with_git_lfs.md) - Set up and try [Git LFS](../../topics/git/lfs/index.md)
- Get to know the [GitLab API](../../api/README.md), its capabilities and shortcomings - Get to know the [GitLab API](../../api/README.md), its capabilities and shortcomings
- Learn how to [migrate from SVN to Git](../../user/project/import/svn.md) - Learn how to [migrate from SVN to Git](../../user/project/import/svn.md)
- Set up [GitLab CI/CD](../../ci/quick_start/README.md) - Set up [GitLab CI/CD](../../ci/quick_start/README.md)
......
doc/user/admin_area/settings/account_and_limit_settings.md
View file @ aa5831c2
...@@ -44,7 +44,7 @@ For instance, consider the following workflow: ...@@ -44,7 +44,7 @@ For instance, consider the following workflow:
1. Your team develops apps which require large files to be stored in 1. Your team develops apps which require large files to be stored in
the application repository. the application repository.
1. Although you have enabled [Git LFS](../../../administration/lfs/manage_large_binaries_with_git_lfs.md#git-lfs) 1. Although you have enabled [Git LFS](../../../topics/git/lfs/index.md#git-lfs)
to your project, your storage has grown significantly. to your project, your storage has grown significantly.
1. Before you exceed available storage, you set up a limit of 10 GB 1. Before you exceed available storage, you set up a limit of 10 GB
per repository. per repository.
......
doc/user/project/issues/design_management.md
View file @ aa5831c2
...@@ -22,7 +22,7 @@ For an overview, see the video [Design Management (GitLab 12.2)](https://www.you ...@@ -22,7 +22,7 @@ For an overview, see the video [Design Management (GitLab 12.2)](https://www.you
## Requirements ## Requirements
Design Management requires Design Management requires
[Large File Storage (LFS)](../../../administration/lfs/manage_large_binaries_with_git_lfs.md) [Large File Storage (LFS)](../../../topics/git/lfs/index.md)
to be enabled: to be enabled:
- For GitLab.com, LFS is already enabled. - For GitLab.com, LFS is already enabled.
......
doc/user/project/settings/index.md
View file @ aa5831c2
...@@ -46,7 +46,7 @@ Use the switches to enable or disable the following features: ...@@ -46,7 +46,7 @@ Use the switches to enable or disable the following features:
| **Forks** | ✓ | Enables [forking](../index.md#fork-a-project) functionality | | **Forks** | ✓ | Enables [forking](../index.md#fork-a-project) functionality |
| **Pipelines** | ✓ | Enables [CI/CD](../../../ci/README.md) functionality | | **Pipelines** | ✓ | Enables [CI/CD](../../../ci/README.md) functionality |
| **Container Registry** | | Activates a [registry](../../packages/container_registry/) for your docker images | | **Container Registry** | | Activates a [registry](../../packages/container_registry/) for your docker images |
| **Git Large File Storage** | | Enables the use of [large files](../../../administration/lfs/manage_large_binaries_with_git_lfs.md#git-lfs) | | **Git Large File Storage** | | Enables the use of [large files](../../../topics/git/lfs/index.md#git-lfs) |
| **Packages** | | Supports configuration of a [package registry](../../../administration/packages/index.md#gitlab-package-registry-administration-premium-only) functionality | | **Packages** | | Supports configuration of a [package registry](../../../administration/packages/index.md#gitlab-package-registry-administration-premium-only) functionality |
| **Wiki** | ✓ | Enables a separate system for [documentation](../wiki/) | | **Wiki** | ✓ | Enables a separate system for [documentation](../wiki/) |
| **Snippets** | ✓ | Enables [sharing of code and text](../../snippets.md) | | **Snippets** | ✓ | Enables [sharing of code and text](../../snippets.md) |
......
doc/workflow/git_lfs.md
View file @ aa5831c2
--- ---
redirect_to: '../administration/lfs/manage_large_binaries_with_git_lfs.md' redirect_to: '../topics/git/lfs/index.md'
--- ---
This document was moved to [another location](../administration/lfs/manage_large_binaries_with_git_lfs.md). This document was moved to [another location](../topics/git/lfs/index.md).
doc/workflow/lfs/manage_large_binaries_with_git_lfs.md
View file @ aa5831c2
--- ---
redirect_to: '../../administration/lfs/manage_large_binaries_with_git_lfs.md' redirect_to: '../../topics/git/lfs/index.md'
--- ---
This document was moved to [another location](../../administration/lfs/manage_large_binaries_with_git_lfs.md). This document was moved to [another location](../../topics/git/lfs/index.md).
doc/workflow/lfs/migrate_from_git_annex_to_git_lfs.md
View file @ aa5831c2
--- ---
redirect_to: '../../administration/lfs/migrate_from_git_annex_to_git_lfs.md' redirect_to: '../../topics/git/lfs/migrate_from_git_annex_to_git_lfs.md'
--- ---
This document was moved to [another location](../../administration/lfs/migrate_from_git_annex_to_git_lfs.md). This document was moved to [another location](../../topics/git/lfs/migrate_from_git_annex_to_git_lfs.md).
ee/app/helpers/ee/mirror_helper.rb
View file @ aa5831c2
...@@ -38,7 +38,7 @@ module EE ...@@ -38,7 +38,7 @@ module EE
end end
def mirror_lfs_sync_message def mirror_lfs_sync_message
docs_link_url = help_page_path('workflow/lfs/manage_large_binaries_with_git_lfs') docs_link_url = help_page_path('topics/git/lfs/index')
docs_link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: docs_link_url } docs_link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: docs_link_url }
_('Git LFS objects will be synced in pull mirrors if LFS is %{docs_link_start}enabled for the project%{docs_link_end}. They will <strong>not</strong> be synced in push mirrors.').html_safe % { docs_link_start: docs_link_start, docs_link_end: '</a>'.html_safe } _('Git LFS objects will be synced in pull mirrors if LFS is %{docs_link_start}enabled for the project%{docs_link_end}. They will <strong>not</strong> be synced in push mirrors.').html_safe % { docs_link_start: docs_link_start, docs_link_end: '</a>'.html_safe }
......
spec/frontend/pages/projects/shared/permissions/components/settings_panel_spec.js
View file @ aa5831c2
...@@ -31,9 +31,9 @@ const defaultProps = { ...@@ -31,9 +31,9 @@ const defaultProps = {
registryAvailable: false, registryAvailable: false,
registryHelpPath: '/help/user/packages/container_registry/index', registryHelpPath: '/help/user/packages/container_registry/index',
lfsAvailable: true, lfsAvailable: true,
lfsHelpPath: '/help/workflow/lfs/manage_large_binaries_with_git_lfs', lfsHelpPath: '/help/topics/git/lfs/index',
lfsObjectsExist: false, lfsObjectsExist: false,
lfsObjectsRemovalHelpPath: `/help/administration/lfs/manage_large_binaries_with_git_lfs#removing-objects-from-lfs`, lfsObjectsRemovalHelpPath: `/help/topics/git/lfs/index#removing-objects-from-lfs`,
pagesAvailable: true, pagesAvailable: true,
pagesAccessControlEnabled: false, pagesAccessControlEnabled: false,
pagesAccessControlForced: false, pagesAccessControlForced: false,
...@@ -363,7 +363,7 @@ describe('Settings Panel', () => { ...@@ -363,7 +363,7 @@ describe('Settings Panel', () => {
); );
expect(link.text()).toEqual('How do I remove them?'); expect(link.text()).toEqual('How do I remove them?');
expect(link.attributes('href')).toEqual( expect(link.attributes('href')).toEqual(
'/help/administration/lfs/manage_large_binaries_with_git_lfs#removing-objects-from-lfs', '/help/topics/git/lfs/index#removing-objects-from-lfs',
); );
}); });
} else { } else {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7