Merge branch 'import-gh-wiki' into 'master'

Import GitHub wiki into GitLab

Closes #2834 

GitHub doesn’t apply any constraints to theirs wiki slug allowing chars like ,, :, *, etc, we need to remove our constraints or some wiki pages will not be available after they are imported.

Some wikis use the Gollum's tags to link its internal/external resources like: images, urls, wiki pages, etc. So, to avoid that wiki links/images displays completely broken after they were imported, we added a new `WikiPipeline`, that for now will parse only simple links, and image tags.

##### Before `WikiPipeline`:

![Screenshot_2016-01-11_20.14.48](/uploads/46fd5dbb5acfc70aa8ecca3050b675e4/Screenshot_2016-01-11_20.14.48.png)

##### After `WikiPipeline`:

![Screenshot_2016-01-11_20.15.56](/uploads/b1d94aa852f385f867a7868c358b4257/Screenshot_2016-01-11_20.15.56.png)

See merge request !2324

CHANGELOG

@@ -50,6 +50,7 @@ v 8.4.0 (unreleased)
   - Fix Encoding::CompatibilityError bug when markdown content has some complex URL (Jason Lee)
   - Add API support for managing build variables of project
   - Allow broadcast messages to be edited
+  - Import GitHub wiki into GitLab
 
 v 8.3.4
   - Use gitlab-workhorse 0.5.4 (fixes API routing bug)

app/assets/javascripts/wikis.js.coffee

+#= require latinise
+
 class @Wikis
   constructor: ->
-    $('.build-new-wiki').bind "click", (e) ->
-      $('[data-error~=slug]').addClass("hidden")
-      $('p.hint').show()
+    $('.build-new-wiki').bind 'click', (e) =>
+      $('[data-error~=slug]').addClass('hidden')
       field = $('#new_wiki_path')
-      valid_slug_pattern = /^[\w\/-]+$/
+      slug = @slugify(field.val())
 
-      slug = field.val()
-      if slug.match valid_slug_pattern
+      if (slug.length > 0)
         path = field.attr('data-wikis-path')
-        if(slug.length > 0)
-          location.href = path + "/" + slug
-      else
-        e.preventDefault()
-        $('p.hint').hide()
-        $('[data-error~=slug]').removeClass("hidden")
+        location.href = path + '/' + slug
+
+  dasherize: (value) ->
+    value.replace(/[_\s]+/g, '-')
+
+  slugify: (value) =>
+    @dasherize(value.trim().toLowerCase().latinise())

app/helpers/gitlab_markdown_helper.rb

@@ -91,7 +91,7 @@ module GitlabMarkdownHelper
   def render_wiki_content(wiki_page)
     case wiki_page.format
     when :markdown
-      markdown(wiki_page.content)
+      markdown(wiki_page.content, pipeline: :wiki, project_wiki: @project_wiki)
     when :asciidoc
       asciidoc(wiki_page.content)
     else

app/models/project_wiki.rb

@@ -38,6 +38,10 @@ class ProjectWiki
     [Gitlab.config.gitlab.url, "/", path_with_namespace, ".git"].join('')
   end
 
+  def wiki_base_path
+    ["/", @project.path_with_namespace, "/wikis"].join('')
+  end
+
   # Returns the Gollum::Wiki object.
   def wiki
     @wiki ||= begin

app/models/wiki_page.rb

@@ -169,7 +169,7 @@ class WikiPage
   private
 
   def set_attributes
-    attributes[:slug] = @page.escaped_url_path
+    attributes[:slug] = @page.url_path
     attributes[:title] = @page.title
     attributes[:format] = @page.format
   end

app/views/projects/wikis/_new.html.haml

@@ -5,12 +5,9 @@
         %a.close{href: "#", "data-dismiss" => "modal"} ×
         %h3.page-title New Wiki Page
       .modal-body
+        .form-group
           = label_tag :new_wiki_path do
             %span Page slug
           = text_field_tag :new_wiki_path, nil, placeholder: 'how-to-setup', class: 'form-control', required: true, :'data-wikis-path' => namespace_project_wikis_path(@project.namespace, @project)
-        %p.hidden.text-danger{data: { error: "slug" }}
-          The page slug is invalid. Please don't use characters other then: a-z 0-9 _ - and /
-        %p.hint
-          Please don't use spaces.
         .form-actions
           = link_to 'Create Page', '#', class: 'build-new-wiki btn btn-create'

config/routes.rb

@@ -519,7 +519,7 @@ Rails.application.routes.draw do
           end
         end
 
-        WIKI_SLUG_ID = { id: /[a-zA-Z.0-9_\-\/]+/ } unless defined? WIKI_SLUG_ID
+        WIKI_SLUG_ID = { id: /\S+/ } unless defined? WIKI_SLUG_ID
 
         scope do
           # Order matters to give priority to these matches

features/project/wiki.feature

@@ -69,11 +69,6 @@ Feature: Project Wiki
     And I click on the "Pages" button
     Then I should see non-escaped link in the pages list
 
-  @javascript
-  Scenario: Creating an invalid new page
-    Given I create a New page with an invalid name
-    Then I should see an error message
-
   @javascript
   Scenario: Edit Wiki page that has a path
     Given I create a New page with paths

features/steps/project/wiki.rb

@@ -132,16 +132,6 @@ class Spinach::Features::ProjectWiki < Spinach::FeatureSteps
     expect(current_path).to include 'one/two/three'
   end
 
-  step 'I create a New page with an invalid name' do
-    click_on 'New Page'
-    fill_in 'Page slug', with: 'invalid name'
-    click_on 'Create Page'
-  end
-
-  step 'I should see an error message' do
-    expect(page).to have_content "The page slug is invalid"
-  end
-
   step 'I should see non-escaped link in the pages list' do
     expect(page).to have_xpath("//a[@href='/#{project.path_with_namespace}/wikis/one/two/three']")
   end

lib/banzai/filter/gollum_tags_filter.rb

+require 'banzai'
+require 'html/pipeline/filter'
+
+module Banzai
+  module Filter
+    # HTML Filter for parsing Gollum's tags in HTML. It's only parses the
+    # following tags:
+    #
+    # - Link to internal pages:
+    #
+    #   * [[Bug Reports]]
+    #   * [[How to Contribute|Contributing]]
+    #
+    # - Link to external resources:
+    #
+    #   * [[http://en.wikipedia.org/wiki/Git_(software)]]
+    #   * [[Git|http://en.wikipedia.org/wiki/Git_(software)]]
+    #
+    # - Link internal images, the special attributes will be ignored:
+    #
+    #   * [[images/logo.png]]
+    #   * [[images/logo.png|alt=Logo]]
+    #
+    # - Link external images, the special attributes will be ignored:
+    #
+    #   * [[http://example.com/images/logo.png]]
+    #   * [[http://example.com/images/logo.png|alt=Logo]]
+    #
+    # Based on Gollum::Filter::Tags
+    #
+    # Context options:
+    #   :project_wiki (required) - Current project wiki.
+    #
+    class GollumTagsFilter < HTML::Pipeline::Filter
+      include ActionView::Helpers::TagHelper
+
+      # Pattern to match tags content that should be parsed in HTML.
+      #
+      # Gollum's tags have been made to resemble the tags of other markups,
+      # especially MediaWiki. The basic syntax is:
+      #
+      # [[tag]]
+      #
+      # Some tags will accept attributes which are separated by pipe
+      # symbols.Some attributes must precede the tag and some must follow it:
+      #
+      # [[prefix-attribute|tag]]
+      # [[tag|suffix-attribute]]
+      #
+      # See https://github.com/gollum/gollum/wiki
+      #
+      # Rubular: http://rubular.com/r/7dQnE5CUCH
+      TAGS_PATTERN = %r{\[\[(.+?)\]\]}
+
+      # Pattern to match allowed image extensions
+      ALLOWED_IMAGE_EXTENSIONS = %r{.+(jpg|png|gif|svg|bmp)\z}i
+
+      def call
+        search_text_nodes(doc).each do |node|
+          content = node.content
+
+          next unless content.match(TAGS_PATTERN)
+
+          html = process_tag($1)
+
+          if html && html != node.content
+            node.replace(html)
+          end
+        end
+
+        doc
+      end
+
+      private
+
+      # Process a single tag into its final HTML form.
+      #
+      # tag - The String tag contents (the stuff inside the double brackets).
+      #
+      # Returns the String HTML version of the tag.
+      def process_tag(tag)
+        parts = tag.split('|')
+
+        return if parts.size.zero?
+
+        process_image_tag(parts) || process_page_link_tag(parts)
+      end
+
+      # Attempt to process the tag as an image tag.
+      #
+      # tag - The String tag contents (the stuff inside the double brackets).
+      #
+      # Returns the String HTML if the tag is a valid image tag or nil
+      # if it is not.
+      def process_image_tag(parts)
+        content = parts[0].strip
+
+        return unless image?(content)
+
+        if url?(content)
+          path = content
+        elsif file = project_wiki.find_file(content)
+          path = ::File.join project_wiki_base_path, file.path
+        end
+
+        if path
+          content_tag(:img, nil, src: path)
+        end
+      end
+
+      def image?(path)
+        path =~ ALLOWED_IMAGE_EXTENSIONS
+      end
+
+      def url?(path)
+        path.start_with?(*%w(http https))
+      end
+
+      # Attempt to process the tag as a page link tag.
+      #
+      # tag - The String tag contents (the stuff inside the double brackets).
+      #
+      # Returns the String HTML if the tag is a valid page link tag or nil
+      # if it is not.
+      def process_page_link_tag(parts)
+        if parts.size == 1
+          url = parts[0].strip
+        else
+          name, url = *parts.compact.map(&:strip)
+        end
+
+        content_tag(:a, name || url, href: url)
+      end
+
+      def project_wiki
+        context[:project_wiki]
+      end
+
+      def project_wiki_base_path
+        project_wiki && project_wiki.wiki_base_path
+      end
+
+      # Ensure that a :project_wiki key exists in context
+      #
+      # Note that while the key might exist, its value could be nil!
+      def validate
+        needs :project_wiki
+      end
+    end
+  end
+end

lib/banzai/pipeline/wiki_pipeline.rb

+require 'banzai'
+
+module Banzai
+  module Pipeline
+    class WikiPipeline < FullPipeline
+      def self.filters
+        super.insert(1, Filter::GollumTagsFilter)
+      end
+    end
+  end
+end

lib/gitlab/github_import/importer.rb

 module Gitlab
   module GithubImport
     class Importer
+      include Gitlab::ShellAdapter
+
       attr_reader :project, :client
 
       def initialize(project)
@@ -12,10 +14,7 @@ module Gitlab
       end
 
       def execute
-        import_issues
-        import_pull_requests
-
-        true
+        import_issues && import_pull_requests && import_wiki
       end
 
       private
@@ -34,6 +33,10 @@ module Gitlab
             end
           end
         end
+
+        true
+      rescue ActiveRecord::RecordInvalid
+        false
       end
 
       def import_pull_requests
@@ -48,6 +51,10 @@ module Gitlab
             import_comments_on_diff(pull_request.number, merge_request)
           end
         end
+
+        true
+      rescue ActiveRecord::RecordInvalid
+        false
       end
 
       def import_comments(issue_number, noteable)
@@ -66,6 +73,18 @@ module Gitlab
           noteable.notes.create!(comment.attributes)
         end
       end
+
+      def import_wiki
+        unless project.wiki_enabled?
+          wiki = WikiFormatter.new(project)
+          gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url)
+          project.update_attribute(:wiki_enabled, true)
+        end
+
+        true
+      rescue Gitlab::Shell::Error
+        false
+      end
     end
   end
 end

lib/gitlab/github_import/project_creator.rb

@@ -20,7 +20,8 @@ module Gitlab
           visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC,
           import_type: "github",
           import_source: repo.full_name,
-          import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@")
+          import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@"),
+          wiki_enabled: !repo.has_wiki? # If repo has wiki we'll import it later
         ).execute
 
         project.create_import_data(data: { "github_session" => session_data } )

lib/gitlab/github_import/wiki_formatter.rb

+module Gitlab
+  module GithubImport
+    class WikiFormatter
+      attr_reader :project
+
+      def initialize(project)
+        @project = project
+      end
+
+      def path_with_namespace
+        "#{project.path_with_namespace}.wiki"
+      end
+
+      def import_url
+        project.import_url.sub(/\.git\z/, ".wiki.git")
+      end
+    end
+  end
+end

spec/features/markdown_spec.rb

@@ -175,13 +175,15 @@ describe 'GitLab Markdown', feature: true do
     end
   end
 
-  context 'default pipeline' do
   before(:all) do
     @feat = MarkdownFeature.new
 
     # `markdown` helper expects a `@project` variable
     @project = @feat.project
+  end
 
+  context 'default pipeline' do
+    before(:all) do
       @html = markdown(@feat.raw_markdown)
     end
 
@@ -221,6 +223,57 @@ describe 'GitLab Markdown', feature: true do
     end
   end
 
+  context 'wiki pipeline' do
+    before do
+      @project_wiki = @feat.project_wiki
+
+      file = Gollum::File.new(@project_wiki.wiki)
+      expect(file).to receive(:path).and_return('images/example.jpg')
+      expect(@project_wiki).to receive(:find_file).with('images/example.jpg').and_return(file)
+
+      @html = markdown(@feat.raw_markdown, { pipeline: :wiki, project_wiki: @project_wiki })
+    end
+
+    it_behaves_like 'all pipelines'
+
+    it 'includes RelativeLinkFilter' do
+      expect(doc).not_to parse_relative_links
+    end
+
+    it 'includes EmojiFilter' do
+      expect(doc).to parse_emoji
+    end
+
+    it 'includes TableOfContentsFilter' do
+      expect(doc).to create_header_links
+    end
+
+    it 'includes AutolinkFilter' do
+      expect(doc).to create_autolinks
+    end
+
+    it 'includes all reference filters' do
+      aggregate_failures do
+        expect(doc).to reference_users
+        expect(doc).to reference_issues
+        expect(doc).to reference_merge_requests
+        expect(doc).to reference_snippets
+        expect(doc).to reference_commit_ranges
+        expect(doc).to reference_commits
+        expect(doc).to reference_labels
+        expect(doc).to reference_milestones
+      end
+    end
+
+    it 'includes TaskListFilter' do
+      expect(doc).to parse_task_lists
+    end
+
+    it 'includes GollumTagsFilter' do
+      expect(doc).to parse_gollum_tags
+    end
+  end
+
   # Fake a `current_user` helper
   def current_user
     @feat.user

spec/fixtures/markdown.md.erb

@@ -230,3 +230,12 @@ References should be parseable even inside _<%= merge_request.to_reference %>_ e
   - [ ] Incomplete sub-task 2
   - [x] Complete sub-task 1
 - [X] Complete task 2
+
+#### Gollum Tags
+
+- [[linked-resource]]
+- [[link-text|linked-resource]]
+- [[http://example.com]]
+- [[link-text|http://example.com/pdfs/gollum.pdf]]
+- [[images/example.jpg]]
+- [[http://example.com/images/example.jpg]]

spec/helpers/gitlab_markdown_helper_spec.rb

@@ -121,12 +121,13 @@ describe GitlabMarkdownHelper do
     before do
       @wiki = double('WikiPage')
       allow(@wiki).to receive(:content).and_return('wiki content')
+      helper.instance_variable_set(:@project_wiki, @wiki)
     end
 
-    it "should use GitLab Flavored Markdown for markdown files" do
+    it "should use Wiki pipeline for markdown files" do
       allow(@wiki).to receive(:format).and_return(:markdown)
 
-      expect(helper).to receive(:markdown).with('wiki content')
+      expect(helper).to receive(:markdown).with('wiki content', pipeline: :wiki, project_wiki: @wiki)
 
       helper.render_wiki_content(@wiki)
     end

spec/lib/banzai/filter/gollum_tags_filter_spec.rb

+require 'spec_helper'
+
+describe Banzai::Filter::GollumTagsFilter, lib: true do
+  include FilterSpecHelper
+
+  let(:project) { create(:project) }
+  let(:user) { double }
+  let(:project_wiki) { ProjectWiki.new(project, user) }
+
+  describe 'validation' do
+    it 'ensure that a :project_wiki key exists in context' do
+      expect { filter("See [[images/image.jpg]]", {}) }.to raise_error ArgumentError, "Missing context keys for Banzai::Filter::GollumTagsFilter: :project_wiki"
+    end
+  end
+
+  context 'linking internal images' do
+    it 'creates img tag if image exists' do
+      file = Gollum::File.new(project_wiki.wiki)
+      expect(file).to receive(:path).and_return('images/image.jpg')
+      expect(project_wiki).to receive(:find_file).with('images/image.jpg').and_return(file)
+
+      tag = '[[images/image.jpg]]'
+      doc = filter("See #{tag}", project_wiki: project_wiki)
+
+      expect(doc.at_css('img')['src']).to eq "#{project_wiki.wiki_base_path}/images/image.jpg"
+    end
+
+    it 'does not creates img tag if image does not exist' do
+      expect(project_wiki).to receive(:find_file).with('images/image.jpg').and_return(nil)
+
+      tag = '[[images/image.jpg]]'
+      doc = filter("See #{tag}", project_wiki: project_wiki)
+
+      expect(doc.css('img').size).to eq 0
+    end
+  end
+
+  context 'linking external images' do
+    it 'creates img tag for valid URL' do
+      tag = '[[http://example.com/image.jpg]]'
+      doc = filter("See #{tag}", project_wiki: project_wiki)
+
+      expect(doc.at_css('img')['src']).to eq "http://example.com/image.jpg"
+    end
+
+    it 'does not creates img tag for invalid URL' do
+      tag = '[[http://example.com/image.pdf]]'
+      doc = filter("See #{tag}", project_wiki: project_wiki)
+
+      expect(doc.css('img').size).to eq 0
+    end
+  end
+
+  context 'linking external resources' do
+    it "the created link's text will be equal to the resource's text" do
+      tag = '[[http://example.com]]'
+      doc = filter("See #{tag}", project_wiki: project_wiki)
+
+      expect(doc.at_css('a').text).to eq 'http://example.com'
+      expect(doc.at_css('a')['href']).to eq 'http://example.com'
+    end
+
+    it "the created link's text will be link-text" do
+      tag = '[[link-text|http://example.com/pdfs/gollum.pdf]]'
+      doc = filter("See #{tag}", project_wiki: project_wiki)
+
+      expect(doc.at_css('a').text).to eq 'link-text'
+      expect(doc.at_css('a')['href']).to eq 'http://example.com/pdfs/gollum.pdf'
+    end
+  end
+
+  context 'linking internal resources' do
+    it "the created link's text will be equal to the resource's text" do
+      tag = '[[wiki-slug]]'
+      doc = filter("See #{tag}", project_wiki: project_wiki)
+
+      expect(doc.at_css('a').text).to eq 'wiki-slug'
+      expect(doc.at_css('a')['href']).to eq 'wiki-slug'
+    end
+
+    it "the created link's text will be link-text" do
+      tag = '[[link-text|wiki-slug]]'
+      doc = filter("See #{tag}", project_wiki: project_wiki)
+
+      expect(doc.at_css('a').text).to eq 'link-text'
+      expect(doc.at_css('a')['href']).to eq 'wiki-slug'
+    end
+  end
+end

spec/lib/gitlab/github_import/wiki_formatter_spec.rb

+require 'spec_helper'
+
+describe Gitlab::GithubImport::WikiFormatter, lib: true do
+  let(:project) do
+    create(:project, namespace: create(:namespace, path: 'gitlabhq'),
+                     import_url: 'https://xxx@github.com/gitlabhq/sample.gitlabhq.git')
+  end
+
+  subject(:wiki) { described_class.new(project)}
+
+  describe '#path_with_namespace' do
+    it 'appends .wiki to project path' do
+      expect(wiki.path_with_namespace).to eq 'gitlabhq/gitlabhq.wiki'
+    end
+  end
+
+  describe '#import_url' do
+    it 'returns URL of the wiki repository' do
+      expect(wiki.import_url).to eq 'https://xxx@github.com/gitlabhq/sample.gitlabhq.wiki.git'
+    end
+  end
+end

spec/models/project_wiki_spec.rb

@@ -36,6 +36,13 @@ describe ProjectWiki, models: true do
     end
   end
 
+  describe "#wiki_base_path" do
+    it "returns the wiki base path" do
+      wiki_base_path = "/#{project.path_with_namespace}/wikis"
+      expect(subject.wiki_base_path).to eq(wiki_base_path)
+    end
+  end
+
   describe "#wiki" do
     it "contains a Gollum::Wiki instance" do
       expect(subject.wiki).to be_a Gollum::Wiki

spec/support/markdown_feature.rb

@@ -28,6 +28,10 @@ class MarkdownFeature
     end
   end
 
+  def project_wiki
+    @project_wiki ||= ProjectWiki.new(project, user)
+  end
+
   def issue
     @issue ||= create(:issue, project: project)
   end

spec/support/matchers/markdown_matchers.rb

@@ -66,6 +66,24 @@ module MarkdownMatchers
     end
   end
 
+  # GollumTagsFilter
+  matcher :parse_gollum_tags do
+    def have_image(src)
+      have_css("img[src$='#{src}']")
+    end
+
+    set_default_markdown_messages
+
+    match do |actual|
+      expect(actual).to have_link('linked-resource', href: 'linked-resource')
+      expect(actual).to have_link('link-text', href: 'linked-resource')
+      expect(actual).to have_link('http://example.com', href: 'http://example.com')
+      expect(actual).to have_link('link-text', href: 'http://example.com/pdfs/gollum.pdf')
+      expect(actual).to have_image('/gitlabhq/wikis/images/example.jpg')
+      expect(actual).to have_image('http://example.com/images/example.jpg')
+    end
+  end
+
   # UserReferenceFilter
   matcher :reference_users do
     set_default_markdown_messages

vendor/assets/javascripts/latinise.js

+// Converting text to basic latin (aka removing accents)
+//
+// Based on: http://semplicewebsites.com/removing-accents-javascript
+//
+var Latinise = {
+  map: {"Á":"A","Ă":"A","Ắ":"A","Ặ":"A","Ằ":"A","Ẳ":"A","Ẵ":"A","Ǎ":"A","Â":"A","Ấ":"A","Ậ":"A","Ầ":"A","Ẩ":"A","Ẫ":"A","Ä":"A","Ǟ":"A","Ȧ":"A","Ǡ":"A","Ạ":"A","Ȁ":"A","À":"A","Ả":"A","Ȃ":"A","Ā":"A","Ą":"A","Å":"A","Ǻ":"A","Ḁ":"A","Ⱥ":"A","Ã":"A","Ꜳ":"AA","Æ":"AE","Ǽ":"AE","Ǣ":"AE","Ꜵ":"AO","Ꜷ":"AU","Ꜹ":"AV","Ꜻ":"AV","Ꜽ":"AY","Ḃ":"B","Ḅ":"B","Ɓ":"B","Ḇ":"B","Ƀ":"B","Ƃ":"B","Ć":"C","Č":"C","Ç":"C","Ḉ":"C","Ĉ":"C","Ċ":"C","Ƈ":"C","Ȼ":"C","Ď":"D","Ḑ":"D","Ḓ":"D","Ḋ":"D","Ḍ":"D","Ɗ":"D","Ḏ":"D","Dz":"D","Dž":"D","Đ":"D","Ƌ":"D","DZ":"DZ","DŽ":"DZ","É":"E","Ĕ":"E","Ě":"E","Ȩ":"E","Ḝ":"E","Ê":"E","Ế":"E","Ệ":"E","Ề":"E","Ể":"E","Ễ":"E","Ḙ":"E","Ë":"E","Ė":"E","Ẹ":"E","Ȅ":"E","È":"E","Ẻ":"E","Ȇ":"E","Ē":"E","Ḗ":"E","Ḕ":"E","Ę":"E","Ɇ":"E","Ẽ":"E","Ḛ":"E","Ꝫ":"ET","Ḟ":"F","Ƒ":"F","Ǵ":"G","Ğ":"G","Ǧ":"G","Ģ":"G","Ĝ":"G","Ġ":"G","Ɠ":"G","Ḡ":"G","Ǥ":"G","Ḫ":"H","Ȟ":"H","Ḩ":"H","Ĥ":"H","Ⱨ":"H","Ḧ":"H","Ḣ":"H","Ḥ":"H","Ħ":"H","Í":"I","Ĭ":"I","Ǐ":"I","Î":"I","Ï":"I","Ḯ":"I","İ":"I","Ị":"I","Ȉ":"I","Ì":"I","Ỉ":"I","Ȋ":"I","Ī":"I","Į":"I","Ɨ":"I","Ĩ":"I","Ḭ":"I","Ꝺ":"D","Ꝼ":"F","Ᵹ":"G","Ꞃ":"R","Ꞅ":"S","Ꞇ":"T","Ꝭ":"IS","Ĵ":"J","Ɉ":"J","Ḱ":"K","Ǩ":"K","Ķ":"K","Ⱪ":"K","Ꝃ":"K","Ḳ":"K","Ƙ":"K","Ḵ":"K","Ꝁ":"K","Ꝅ":"K","Ĺ":"L","Ƚ":"L","Ľ":"L","Ļ":"L","Ḽ":"L","Ḷ":"L","Ḹ":"L","Ⱡ":"L","Ꝉ":"L","Ḻ":"L","Ŀ":"L","Ɫ":"L","Lj":"L","Ł":"L","LJ":"LJ","Ḿ":"M","Ṁ":"M","Ṃ":"M","Ɱ":"M","Ń":"N","Ň":"N","Ņ":"N","Ṋ":"N","Ṅ":"N","Ṇ":"N","Ǹ":"N","Ɲ":"N","Ṉ":"N","Ƞ":"N","Nj":"N","Ñ":"N","NJ":"NJ","Ó":"O","Ŏ":"O","Ǒ":"O","Ô":"O","Ố":"O","Ộ":"O","Ồ":"O","Ổ":"O","Ỗ":"O","Ö":"O","Ȫ":"O","Ȯ":"O","Ȱ":"O","Ọ":"O","Ő":"O","Ȍ":"O","Ò":"O","Ỏ":"O","Ơ":"O","Ớ":"O","Ợ":"O","Ờ":"O","Ở":"O","Ỡ":"O","Ȏ":"O","Ꝋ":"O","Ꝍ":"O","Ō":"O","Ṓ":"O","Ṑ":"O","Ɵ":"O","Ǫ":"O","Ǭ":"O","Ø":"O","Ǿ":"O","Õ":"O","Ṍ":"O","Ṏ":"O","Ȭ":"O","Ƣ":"OI","Ꝏ":"OO","Ɛ":"E","Ɔ":"O","Ȣ":"OU","Ṕ":"P","Ṗ":"P","Ꝓ":"P","Ƥ":"P","Ꝕ":"P","Ᵽ":"P","Ꝑ":"P","Ꝙ":"Q","Ꝗ":"Q","Ŕ":"R","Ř":"R","Ŗ":"R","Ṙ":"R","Ṛ":"R","Ṝ":"R","Ȑ":"R","Ȓ":"R","Ṟ":"R","Ɍ":"R","Ɽ":"R","Ꜿ":"C","Ǝ":"E","Ś":"S","Ṥ":"S","Š":"S","Ṧ":"S","Ş":"S","Ŝ":"S","Ș":"S","Ṡ":"S","Ṣ":"S","Ṩ":"S","ẞ":"SS","Ť":"T","Ţ":"T","Ṱ":"T","Ț":"T","Ⱦ":"T","Ṫ":"T","Ṭ":"T","Ƭ":"T","Ṯ":"T","Ʈ":"T","Ŧ":"T","Ɐ":"A","Ꞁ":"L","Ɯ":"M","Ʌ":"V","Ꜩ":"TZ","Ú":"U","Ŭ":"U","Ǔ":"U","Û":"U","Ṷ":"U","Ü":"U","Ǘ":"U","Ǚ":"U","Ǜ":"U","Ǖ":"U","Ṳ":"U","Ụ":"U","Ű":"U","Ȕ":"U","Ù":"U","Ủ":"U","Ư":"U","Ứ":"U","Ự":"U","Ừ":"U","Ử":"U","Ữ":"U","Ȗ":"U","Ū":"U","Ṻ":"U","Ų":"U","Ů":"U","Ũ":"U","Ṹ":"U","Ṵ":"U","Ꝟ":"V","Ṿ":"V","Ʋ":"V","Ṽ":"V","Ꝡ":"VY","Ẃ":"W","Ŵ":"W","Ẅ":"W","Ẇ":"W","Ẉ":"W","Ẁ":"W","Ⱳ":"W","Ẍ":"X","Ẋ":"X","Ý":"Y","Ŷ":"Y","Ÿ":"Y","Ẏ":"Y","Ỵ":"Y","Ỳ":"Y","Ƴ":"Y","Ỷ":"Y","Ỿ":"Y","Ȳ":"Y","Ɏ":"Y","Ỹ":"Y","Ź":"Z","Ž":"Z","Ẑ":"Z","Ⱬ":"Z","Ż":"Z","Ẓ":"Z","Ȥ":"Z","Ẕ":"Z","Ƶ":"Z","IJ":"IJ","Œ":"OE","ᴀ":"A","ᴁ":"AE","ʙ":"B","ᴃ":"B","ᴄ":"C","ᴅ":"D","ᴇ":"E","ꜰ":"F","ɢ":"G","ʛ":"G","ʜ":"H","ɪ":"I","ʁ":"R","ᴊ":"J","ᴋ":"K","ʟ":"L","ᴌ":"L","ᴍ":"M","ɴ":"N","ᴏ":"O","ɶ":"OE","ᴐ":"O","ᴕ":"OU","ᴘ":"P","ʀ":"R","ᴎ":"N","ᴙ":"R","ꜱ":"S","ᴛ":"T","ⱻ":"E","ᴚ":"R","ᴜ":"U","ᴠ":"V","ᴡ":"W","ʏ":"Y","ᴢ":"Z","á":"a","ă":"a","ắ":"a","ặ":"a","ằ":"a","ẳ":"a","ẵ":"a","ǎ":"a","â":"a","ấ":"a","ậ":"a","ầ":"a","ẩ":"a","ẫ":"a","ä":"a","ǟ":"a","ȧ":"a","ǡ":"a","ạ":"a","ȁ":"a","à":"a","ả":"a","ȃ":"a","ā":"a","ą":"a","ᶏ":"a","ẚ":"a","å":"a","ǻ":"a","ḁ":"a","ⱥ":"a","ã":"a","ꜳ":"aa","æ":"ae","ǽ":"ae","ǣ":"ae","ꜵ":"ao","ꜷ":"au","ꜹ":"av","ꜻ":"av","ꜽ":"ay","ḃ":"b","ḅ":"b","ɓ":"b","ḇ":"b","ᵬ":"b","ᶀ":"b","ƀ":"b","ƃ":"b","ɵ":"o","ć":"c","č":"c","ç":"c","ḉ":"c","ĉ":"c","ɕ":"c","ċ":"c","ƈ":"c","ȼ":"c","ď":"d","ḑ":"d","ḓ":"d","ȡ":"d","ḋ":"d","ḍ":"d","ɗ":"d","ᶑ":"d","ḏ":"d","ᵭ":"d","ᶁ":"d","đ":"d","ɖ":"d","ƌ":"d","ı":"i","ȷ":"j","ɟ":"j","ʄ":"j","dz":"dz","dž":"dz","é":"e","ĕ":"e","ě":"e","ȩ":"e","ḝ":"e","ê":"e","ế":"e","ệ":"e","ề":"e","ể":"e","ễ":"e","ḙ":"e","ë":"e","ė":"e","ẹ":"e","ȅ":"e","è":"e","ẻ":"e","ȇ":"e","ē":"e","ḗ":"e","ḕ":"e","ⱸ":"e","ę":"e","ᶒ":"e","ɇ":"e","ẽ":"e","ḛ":"e","ꝫ":"et","ḟ":"f","ƒ":"f","ᵮ":"f","ᶂ":"f","ǵ":"g","ğ":"g","ǧ":"g","ģ":"g","ĝ":"g","ġ":"g","ɠ":"g","ḡ":"g","ᶃ":"g","ǥ":"g","ḫ":"h","ȟ":"h","ḩ":"h","ĥ":"h","ⱨ":"h","ḧ":"h","ḣ":"h","ḥ":"h","ɦ":"h","ẖ":"h","ħ":"h","ƕ":"hv","í":"i","ĭ":"i","ǐ":"i","î":"i","ï":"i","ḯ":"i","ị":"i","ȉ":"i","ì":"i","ỉ":"i","ȋ":"i","ī":"i","į":"i","ᶖ":"i","ɨ":"i","ĩ":"i","ḭ":"i","ꝺ":"d","ꝼ":"f","ᵹ":"g","ꞃ":"r","ꞅ":"s","ꞇ":"t","ꝭ":"is","ǰ":"j","ĵ":"j","ʝ":"j","ɉ":"j","ḱ":"k","ǩ":"k","ķ":"k","ⱪ":"k","ꝃ":"k","ḳ":"k","ƙ":"k","ḵ":"k","ᶄ":"k","ꝁ":"k","ꝅ":"k","ĺ":"l","ƚ":"l","ɬ":"l","ľ":"l","ļ":"l","ḽ":"l","ȴ":"l","ḷ":"l","ḹ":"l","ⱡ":"l","ꝉ":"l","ḻ":"l","ŀ":"l","ɫ":"l","ᶅ":"l","ɭ":"l","ł":"l","lj":"lj","ſ":"s","ẜ":"s","ẛ":"s","ẝ":"s","ḿ":"m","ṁ":"m","ṃ":"m","ɱ":"m","ᵯ":"m","ᶆ":"m","ń":"n","ň":"n","ņ":"n","ṋ":"n","ȵ":"n","ṅ":"n","ṇ":"n","ǹ":"n","ɲ":"n","ṉ":"n","ƞ":"n","ᵰ":"n","ᶇ":"n","ɳ":"n","ñ":"n","nj":"nj","ó":"o","ŏ":"o","ǒ":"o","ô":"o","ố":"o","ộ":"o","ồ":"o","ổ":"o","ỗ":"o","ö":"o","ȫ":"o","ȯ":"o","ȱ":"o","ọ":"o","ő":"o","ȍ":"o","ò":"o","ỏ":"o","ơ":"o","ớ":"o","ợ":"o","ờ":"o","ở":"o","ỡ":"o","ȏ":"o","ꝋ":"o","ꝍ":"o","ⱺ":"o","ō":"o","ṓ":"o","ṑ":"o","ǫ":"o","ǭ":"o","ø":"o","ǿ":"o","õ":"o","ṍ":"o","ṏ":"o","ȭ":"o","ƣ":"oi","ꝏ":"oo","ɛ":"e","ᶓ":"e","ɔ":"o","ᶗ":"o","ȣ":"ou","ṕ":"p","ṗ":"p","ꝓ":"p","ƥ":"p","ᵱ":"p","ᶈ":"p","ꝕ":"p","ᵽ":"p","ꝑ":"p","ꝙ":"q","ʠ":"q","ɋ":"q","ꝗ":"q","ŕ":"r","ř":"r","ŗ":"r","ṙ":"r","ṛ":"r","ṝ":"r","ȑ":"r","ɾ":"r","ᵳ":"r","ȓ":"r","ṟ":"r","ɼ":"r","ᵲ":"r","ᶉ":"r","ɍ":"r","ɽ":"r","ↄ":"c","ꜿ":"c","ɘ":"e","ɿ":"r","ś":"s","ṥ":"s","š":"s","ṧ":"s","ş":"s","ŝ":"s","ș":"s","ṡ":"s","ṣ":"s","ṩ":"s","ʂ":"s","ᵴ":"s","ᶊ":"s","ȿ":"s","ɡ":"g","ß":"ss","ᴑ":"o","ᴓ":"o","ᴝ":"u","ť":"t","ţ":"t","ṱ":"t","ț":"t","ȶ":"t","ẗ":"t","ⱦ":"t","ṫ":"t","ṭ":"t","ƭ":"t","ṯ":"t","ᵵ":"t","ƫ":"t","ʈ":"t","ŧ":"t","ᵺ":"th","ɐ":"a","ᴂ":"ae","ǝ":"e","ᵷ":"g","ɥ":"h","ʮ":"h","ʯ":"h","ᴉ":"i","ʞ":"k","ꞁ":"l","ɯ":"m","ɰ":"m","ᴔ":"oe","ɹ":"r","ɻ":"r","ɺ":"r","ⱹ":"r","ʇ":"t","ʌ":"v","ʍ":"w","ʎ":"y","ꜩ":"tz","ú":"u","ŭ":"u","ǔ":"u","û":"u","ṷ":"u","ü":"u","ǘ":"u","ǚ":"u","ǜ":"u","ǖ":"u","ṳ":"u","ụ":"u","ű":"u","ȕ":"u","ù":"u","ủ":"u","ư":"u","ứ":"u","ự":"u","ừ":"u","ử":"u","ữ":"u","ȗ":"u","ū":"u","ṻ":"u","ų":"u","ᶙ":"u","ů":"u","ũ":"u","ṹ":"u","ṵ":"u","ᵫ":"ue","ꝸ":"um","ⱴ":"v","ꝟ":"v","ṿ":"v","ʋ":"v","ᶌ":"v","ⱱ":"v","ṽ":"v","ꝡ":"vy","ẃ":"w","ŵ":"w","ẅ":"w","ẇ":"w","ẉ":"w","ẁ":"w","ⱳ":"w","ẘ":"w","ẍ":"x","ẋ":"x","ᶍ":"x","ý":"y","ŷ":"y","ÿ":"y","ẏ":"y","ỵ":"y","ỳ":"y","ƴ":"y","ỷ":"y","ỿ":"y","ȳ":"y","ẙ":"y","ɏ":"y","ỹ":"y","ź":"z","ž":"z","ẑ":"z","ʑ":"z","ⱬ":"z","ż":"z","ẓ":"z","ȥ":"z","ẕ":"z","ᵶ":"z","ᶎ":"z","ʐ":"z","ƶ":"z","ɀ":"z","ff":"ff","ffi":"ffi","ffl":"ffl","fi":"fi","fl":"fl","ij":"ij","œ":"oe","st":"st","ₐ":"a","ₑ":"e","ᵢ":"i","ⱼ":"j","ₒ":"o","ᵣ":"r","ᵤ":"u","ᵥ":"v","ₓ":"x"}
+};
+
+String.prototype.latinise = function() {
+  return this.replace(/[^A-Za-z0-9]/g, function(x) { return Latinise.map[x] || x; });
+};