1e15c..3d6ce (2010-03-09 11:06)
(2010-03-09 22:05) 9241d..4d589
Author: Claes NästénDate: Tue Mar 09 20:06:42 +0100 2010
Subject: Remove unused scm integration scripts and phases route
| integration/scm_git_log_db_add.rb | |
| 0 | @@ -1,253 +0,0 @@ |
| 1 | #!/usr/bin/env ruby |
| 2 | # -*- coding: utf-8 -*- |
| 3 | # |
| 4 | # Copyright © 2007 Claes Nästén <me@pekdon.net> |
| 5 | # |
| 6 | # This file is part of septic. |
| 7 | # |
| 8 | # septic is free software: you can redistribute it and/or modify |
| 9 | # it under the terms of the GNU General Public License as published by |
| 10 | # the Free Software Foundation, version 3 of the License. |
| 11 | # |
| 12 | # septic is distributed in the hope that it will be useful, |
| 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 15 | # GNU General Public License for more details. |
| 16 | # |
| 17 | # You should have received a copy of the GNU General Public License |
| 18 | # along with septic. If not, see <http://www.gnu.org/licenses/>. |
| 19 | # |
| 20 | |
| 21 | require File.join(File.dirname(__FILE__), 'scm_log_db_add') |
| 22 | |
| 23 | # Class handling git log parsing and adding to database |
| 24 | class ScmGitLogDbAdd < ScmLogDbAdd |
| 25 | # Initialize git log |
| 26 | def initialize(path, reference) |
| 27 | super |
| 28 | |
| 29 | # Parse reference, sets where and branch |
| 30 | parse_reference(reference) |
| 31 | end |
| 32 | |
| 33 | def update |
| 34 | raise "can only update to heads not #{@where}" unless @where == 'heads' |
| 35 | |
| 36 | # Get repository for commit |
| 37 | @repository = Repository.find_by_path(@path) |
| 38 | raise "No repository at #{@path}" if @repository.nil? |
| 39 | @revision = @repository.commits.find(:first) |
| 40 | |
| 41 | # Get updates |
| 42 | commits_data = update_branch_log |
| 43 | |
| 44 | # Add commits to database |
| 45 | add_commits(commits_data) |
| 46 | end |
| 47 | |
| 48 | private |
| 49 | # Add commits to database |
| 50 | def add_commits(commits_data) |
| 51 | commits_data.reverse_each do |commit_data| |
| 52 | # Split body / files |
| 53 | commit_data['body'], commit_files = parse_body_files(commit_data['body']) |
| 54 | |
| 55 | # Lookup user |
| 56 | user = get_user_from_email(commit_data['email']) |
| 57 | if user |
| 58 | commit_data['user_id'] = user.id |
| 59 | end |
| 60 | |
| 61 | # Add commit |
| 62 | commit = @repository.commits.create(commit_data) |
| 63 | |
| 64 | # Add commit files |
| 65 | if commit.valid? |
| 66 | commit_files.each do |file| |
| 67 | commit.commit_files.create(file) |
| 68 | end |
| 69 | |
| 70 | # Update commit links |
| 71 | set_revision_relations(commit) |
| 72 | else |
| 73 | puts "failed creating commit with errors %s" % commit.errors.full_messages.join('\n') |
| 74 | end |
| 75 | end |
| 76 | end |
| 77 | |
| 78 | # Update branch log to current heads |
| 79 | def update_branch_log |
| 80 | # Get previous revision |
| 81 | if @revision |
| 82 | @since = "#{@revision.revision}.." |
| 83 | else |
| 84 | @since = '' |
| 85 | end |
| 86 | |
| 87 | # List of commit data |
| 88 | commit = nil |
| 89 | commits_data = [] |
| 90 | |
| 91 | # Run git capturing it's output |
| 92 | gitp = IO.popen("git log -C -M --find-copies-harder --numstat #{@branch} #{@since}heads/#{@branch}") |
| 93 | for line in gitp.readlines |
| 94 | if /^commit / =~ line |
| 95 | # Finish previous commit if any |
| 96 | if not commit.nil? |
| 97 | commits_data << commit |
| 98 | end |
| 99 | |
| 100 | # Initialize new commit |
| 101 | commit = { 'revision' => line['commit '.size,line.size].chomp, |
| 102 | 'body' => [], |
| 103 | 'branch' => @branch } |
| 104 | |
| 105 | elsif not commit.nil? |
| 106 | if /^Author: / =~ line |
| 107 | commit['author'], commit['email'] = parse_author(line['Author: '.size,line.size].chomp) |
| 108 | elsif /^Date: / =~ line |
| 109 | # Get date from line excluding offset if specified with ' +' |
| 110 | date = line.sub(/Date:\s+/, '').chomp |
| 111 | |
| 112 | # Parse and re-format date to a format that most databases |
| 113 | # can handle. |
| 114 | date = DateTime.strptime(date, '%a %b %d %H:%M:%S %Y') |
| 115 | date = date.strftime('%Y-%m-%d %H:%M:%S') |
| 116 | |
| 117 | commit['created_at'] = date |
| 118 | elsif commit.include?('title') |
| 119 | commit['body'] << line.chomp |
| 120 | elsif line.size > 1 |
| 121 | commit['title'] = line.chomp |
| 122 | end |
| 123 | end |
| 124 | end |
| 125 | gitp.close |
| 126 | |
| 127 | # Add last commit |
| 128 | if not commit.nil? |
| 129 | commits_data << commit |
| 130 | end |
| 131 | |
| 132 | commits_data |
| 133 | end |
| 134 | |
| 135 | # Parse body / files part of log message list with body, files |
| 136 | def parse_body_files(body) |
| 137 | check_next = false |
| 138 | file_offset = -1 |
| 139 | |
| 140 | body.each_index do |offset| |
| 141 | if check_next |
| 142 | check_next = false |
| 143 | if /\d+\t\d+\t.*/ =~ body[offset] |
| 144 | file_offset = offset |
| 145 | end |
| 146 | elsif body[offset] == '' |
| 147 | check_next = true |
| 148 | end |
| 149 | end |
| 150 | |
| 151 | if file_offset > -1 |
| 152 | files = [] |
| 153 | body[file_offset,body.size].each do |file_info| |
| 154 | file = parse_body_file(file_info) |
| 155 | files << file unless file.nil? |
| 156 | end |
| 157 | |
| 158 | body = body[1,file_offset-1] |
| 159 | else |
| 160 | files = [] |
| 161 | end |
| 162 | |
| 163 | [body.join("\n"), files] |
| 164 | end |
| 165 | |
| 166 | # Parse single file change |
| 167 | def parse_body_file(file_info) |
| 168 | file_info = file_info.split(/\t+/, 3) |
| 169 | return nil unless file_info.size == 3 |
| 170 | |
| 171 | # Creat new file, default to modify type. |
| 172 | file = { :path => file_info[2], :insertions => file_info[0], :deletions => file_info[1], |
| 173 | :change_type => CommitFile::MODIFY } |
| 174 | |
| 175 | # Detect add, remove, rename or modification of ordinary or binary file. |
| 176 | if file[:insertions] == '-' and file[:deletions] == '-' |
| 177 | # Binary file, they do not get diff stats. |
| 178 | file[:change_type] = CommitFile::BINARY |
| 179 | file[:insertions] = 0 |
| 180 | file[:deletions] = 0 |
| 181 | |
| 182 | elsif file[:path] =~ /^(.*?)\{(.*) => (.*)\}(.*?)$/ |
| 183 | # Rename, can look something like below: |
| 184 | # |
| 185 | # 1 1 app/controllers/{repository_controller.rb => repositories_controller.rb} |
| 186 | # |
| 187 | file[:change_type] = CommitFile::RENAME |
| 188 | file[:path] = "#{$1}#{$2}#{$4}" |
| 189 | file[:path_after] = "#{$1}#{$3}#{$4}" |
| 190 | |
| 191 | else |
| 192 | # Not using an AS clause here, MySQL does not want to play. |
| 193 | lines = ActiveRecord::Base.connection.select_value("SELECT SUM(commit_files.insertions) - SUM(commit_files.deletions) FROM commit_files WHERE commit_files.path = '%s'" % file[:path]) |
| 194 | |
| 195 | if lines.nil? |
| 196 | # New file, not in commit file index |
| 197 | file[:change_type] = CommitFile::ADD |
| 198 | |
| 199 | # On new files without content GIT seems to be confused and renders |
| 200 | # them as renames/copies. Work around this removing the part until => |
| 201 | if file[:path] =~ /^.* => (.*)$/ |
| 202 | file[:path] = $1 |
| 203 | end |
| 204 | |
| 205 | elsif file[:insertions] == '0' and lines.to_i == file[:deletions].to_i |
| 206 | # Removed file |
| 207 | file[:change_type] = CommitFile::REMOVE |
| 208 | end |
| 209 | end |
| 210 | |
| 211 | file |
| 212 | end |
| 213 | |
| 214 | # Parse reference provided by git |
| 215 | def parse_reference(reference) |
| 216 | ref_info = reference.split('/', 3) |
| 217 | raise "reference #{reference} incorrect format" unless ref_info.size == 3 |
| 218 | |
| 219 | @where = ref_info[1] |
| 220 | @branch = ref_info[2] |
| 221 | end |
| 222 | |
| 223 | # Set relations for commit |
| 224 | def set_revision_relations(commit) |
| 225 | # Get previous revision, update with current |
| 226 | prev_commit = @repository.commits.find(:first, :order => 'commits.created_at DESC', :conditions => ['commits.created_at < ?', commit.created_at]) |
| 227 | if prev_commit |
| 228 | prev_commit.commit_next = commit |
| 229 | commit.commit_prev = prev_commit |
| 230 | prev_commit.save |
| 231 | end |
| 232 | |
| 233 | # Get previous revision for branch |
| 234 | prev_branch_commit = @repository.commits.find(:first, :order => 'commits.created_at DESC', :conditions => ['commits.created_at < ? AND commits.branch = ?', commit.created_at, commit.branch]) |
| 235 | if prev_branch_commit |
| 236 | prev_branch_commit.commit_next_branch = commit |
| 237 | commit.commit_prev_branch = prev_branch_commit |
| 238 | prev_branch_commit.save |
| 239 | end |
| 240 | |
| 241 | commit.save |
| 242 | end |
| 243 | end |
| 244 | |
| 245 | if not ENV.include?('GIT_DIR') |
| 246 | raise 'GIT_DIR environment not set' |
| 247 | elsif ARGV.size != 3 |
| 248 | raise 'reference not provided as single argument' |
| 249 | else |
| 250 | log = ScmGitLogDbAdd.new(ENV['GIT_DIR'], ARGV[2]) |
| 251 | |
| 252 | log.update |
| 253 | end |
| config/routes.rb | |
| 27 | @@ -27,7 +27,6 @@ |
| 27 | :subscribe => :get }) |
| 28 | projects.resource(:news) |
| 29 | projects.resource(:activity) |
| 30 | projects.resources(:phases) |
| 31 | projects.resources(:releases) |
| 32 | projects.resource(:repository, |
| 33 | :member => { :browse => :get, |
| ... | |
| integration/scm_svn_log_db_add.rb | |
| 0 | @@ -1,225 +0,0 @@ |
| 1 | #!/usr/bin/env ruby |
| 2 | # -*- coding: utf-8 -*- |
| 3 | # |
| 4 | # Copyright © 2007 Claes Nästén <me@pekdon.net> |
| 5 | # |
| 6 | # This file is part of septic. |
| 7 | # |
| 8 | # septic is free software: you can redistribute it and/or modify |
| 9 | # it under the terms of the GNU General Public License as published by |
| 10 | # the Free Software Foundation, version 3 of the License. |
| 11 | # |
| 12 | # septic is distributed in the hope that it will be useful, |
| 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 15 | # GNU General Public License for more details. |
| 16 | # |
| 17 | # You should have received a copy of the GNU General Public License |
| 18 | # along with septic. If not, see <http://www.gnu.org/licenses/>. |
| 19 | # |
| 20 | |
| 21 | require 'rubygems' |
| 22 | require 'active_record' |
| 23 | require 'yaml' |
| 24 | |
| 25 | require File.join(File.dirname(__FILE__), '..', 'app', 'models', 'repository') |
| 26 | require File.join(File.dirname(__FILE__), '..', 'app', 'models', 'repository_svn') |
| 27 | require File.join(File.dirname(__FILE__), '..', 'app', 'models', 'commit') |
| 28 | require File.join(File.dirname(__FILE__), '..', 'app', 'models', 'commit_svn') |
| 29 | require File.join(File.dirname(__FILE__), '..', 'app', 'models', 'commit_file') |
| 30 | require File.join(File.dirname(__FILE__), '..', 'app', 'models', 'project') |
| 31 | require File.join(File.dirname(__FILE__), '..', 'app', 'models', 'user') |
| 32 | |
| 33 | require File.join(File.dirname(__FILE__), 'scm_log_db_add') |
| 34 | |
| 35 | require 'svn/repos' |
| 36 | require 'svn/info' |
| 37 | |
| 38 | # Class handling git log parsing and adding to database |
| 39 | class ScmSvnLogDbAdd < ScmLogDbAdd |
| 40 | # Update SVN log entries to latest version |
| 41 | def update |
| 42 | # Get repository and current revision |
| 43 | @repository = Repository.find_by_path(@path) |
| 44 | raise "No repository at #{@path}" if @repository.nil? |
| 45 | revision = get_current_revision |
| 46 | |
| 47 | # Update to current version |
| 48 | update_from(revision) |
| 49 | end |
| 50 | |
| 51 | private |
| 52 | |
| 53 | # Get current revision |
| 54 | def get_current_revision |
| 55 | commit = @repository.commits.find(:first) |
| 56 | if commit.nil? |
| 57 | 0 |
| 58 | else |
| 59 | commit.revision.to_i |
| 60 | end |
| 61 | end |
| 62 | |
| 63 | # Update repository starting at revision |
| 64 | def update_from(revision_start) |
| 65 | # Get repository |
| 66 | repo = get_svn_handle |
| 67 | |
| 68 | # Get current revision |
| 69 | revision_end = repo.fs.youngest_rev |
| 70 | |
| 71 | # Go through logs and update the database |
| 72 | (revision_start + 1..revision_end).each do |revision| |
| 73 | add_revision(Svn::Info.new(@path, revision)) |
| 74 | end |
| 75 | end |
| 76 | |
| 77 | # Add single revision to the database |
| 78 | def add_revision(info) |
| 79 | # Get title / body |
| 80 | log = info.log.split("\n", 2) |
| 81 | log = [''] if log.empty? |
| 82 | |
| 83 | # Get base commit data |
| 84 | commit_data = { 'revision' => info.revision, |
| 85 | 'title' => log[0], 'created_at' => info.date.strftime('%Y-%m-%d %H:%M:%S') } |
| 86 | commit_data['body'] = log[1] if log.size > 1 |
| 87 | commit_data['author'], commit_data['email'] = parse_author(info.author) |
| 88 | |
| 89 | # Lookup user |
| 90 | user = get_user_from_email(commit_data['email']) |
| 91 | if user |
| 92 | commit_data['user_id'] = user.id |
| 93 | end |
| 94 | |
| 95 | # Get commit files amd guess branch |
| 96 | commit_files = get_revision_files(info) |
| 97 | commit_data['branch'] = get_branch_name(commit_files) |
| 98 | |
| 99 | # Add data |
| 100 | commit = @repository.commits.create(commit_data) |
| 101 | |
| 102 | # Add commit files |
| 103 | if commit.valid? |
| 104 | commit_files.each do |file| |
| 105 | commit.commit_files.create(file) |
| 106 | end |
| 107 | |
| 108 | # Add relations, this is done after the save as the next pointers are |
| 109 | # to be updated as well. |
| 110 | set_revision_relations(commit) |
| 111 | else |
| 112 | puts "failed creating commit with errors %s" % commit.errors.full_messages.join("\n") |
| 113 | end |
| 114 | end |
| 115 | |
| 116 | # Return name of branch for commit, unknown if unable to guess |
| 117 | def get_branch_name(commit_files) |
| 118 | # Look for /branches/*, /tags/* or /trunk, settle with first match |
| 119 | commit_files.each do |file| |
| 120 | path_info = file[:path].split('/', 3) |
| 121 | next if path_info.size < 2 |
| 122 | |
| 123 | if path_info[0] == 'trunk' |
| 124 | return 'trunk' |
| 125 | elsif path_info.size > 2 |
| 126 | # Only match against tags and branches if second element isn't the |
| 127 | # last (file). |
| 128 | if [0] == 'tags' |
| 129 | return path_info[1] |
| 130 | elsif path_info[0] == 'branches' |
| 131 | return path_info[1] |
| 132 | end |
| 133 | end |
| 134 | end |
| 135 | |
| 136 | 'unknown' |
| 137 | end |
| 138 | |
| 139 | # Return list of files changed in revision |
| 140 | def get_revision_files(info) |
| 141 | commit_files = [] |
| 142 | deleted_files = [] |
| 143 | |
| 144 | # Get modified |
| 145 | info.updated_files.sort.each do |name| |
| 146 | file_info = info.diffs[name][:modified] |
| 147 | commit_files << { :path => name, :insertions => file_info.added_line, :deletions => file_info.deleted_line, |
| 148 | :change_type => CommitFile::MODIFY } |
| 149 | end |
| 150 | |
| 151 | # Get added |
| 152 | info.added_files.sort.each do |name| |
| 153 | commit_files << { :path => name, :insertions => 0, :deletions => 0, |
| 154 | :change_type => CommitFile::ADD } |
| 155 | end |
| 156 | |
| 157 | # Get removed |
| 158 | info.deleted_files.sort.each do |name| |
| 159 | # Add to list of deleted files to aid copy/rename detection |
| 160 | deleted_files << name |
| 161 | commit_files << { :path => name, :insertions => 0, :deletions => 0, |
| 162 | :change_type => CommitFile::REMOVE } |
| 163 | end |
| 164 | |
| 165 | # Get copied/renamed files |
| 166 | info.copied_files.sort.each do |copy_info| |
| 167 | # Check type of change, if file is recorded as deleted this is a rename. |
| 168 | change_type = deleted_files.include?(copy_info[1]) ? CommitFile::RENAME : CommitFile::COPY |
| 169 | |
| 170 | commit_files << { :path => copy_info[1], :path_after => copy_info[0], :insertions => 0, :deletions => 0, |
| 171 | :change_type => change_type } |
| 172 | end |
| 173 | |
| 174 | commit_files |
| 175 | end |
| 176 | |
| 177 | # Set relations for commit |
| 178 | def set_revision_relations(commit) |
| 179 | return if commit.revision.to_i < 2 |
| 180 | |
| 181 | # Get previous revision, update with current |
| 182 | prev_commit = @repository.commits.find(:first, :conditions => ['commits.revision = ?', commit.revision.to_i - 1]) |
| 183 | prev_commit.commit_next = commit |
| 184 | commit.commit_prev = prev_commit |
| 185 | |
| 186 | prev_commit.save |
| 187 | |
| 188 | # Get previous revision for branch |
| 189 | prev_branch_commit = @repository.commits.find(:first, :order => 'commits.created_at DESC', :conditions => ['commits.revision < ? AND commits.branch = ?', commit.revision, commit.branch]) |
| 190 | if prev_branch_commit |
| 191 | prev_branch_commit.commit_next_branch = commit |
| 192 | commit.commit_prev_branch = prev_branch_commit |
| 193 | |
| 194 | prev_branch_commit.save |
| 195 | end |
| 196 | |
| 197 | commit.save |
| 198 | end |
| 199 | |
| 200 | # Get SVN repository handle |
| 201 | def get_svn_handle |
| 202 | if @repo.nil? |
| 203 | @repo = Svn::Repos.open(@path) |
| 204 | end |
| 205 | |
| 206 | return @repo |
| 207 | end |
| 208 | |
| 209 | end |
| 210 | |
| 211 | |
| 212 | cfg = YAML::load_file(File.join(File.dirname(__FILE__), '..', 'config', 'database.yml')) |
| 213 | |
| 214 | mode = ENV.fetch('RAILS_ENV', 'production') |
| 215 | |
| 216 | ActiveRecord::Base.establish_connection(cfg[mode]) |
| 217 | |
| 218 | # Check input parameter and add entries |
| 219 | if ARGV.size != 1 |
| 220 | raise 'svn path not provided as single argument' |
| 221 | else |
| 222 | log = ScmSvnLogDbAdd.new(ARGV[0]) |
| 223 | |
| 224 | log.update |
| 225 | end |
| integration/scm_log_db_add.rb | |
| 0 | @@ -1,72 +0,0 @@ |
| 1 | #!/usr/bin/env ruby |
| 2 | # |
| 3 | # Copyright © 2007 Claes Nästén <me@pekdon.net> |
| 4 | # |
| 5 | # This file is part of septic. |
| 6 | # |
| 7 | # septic is free software: you can redistribute it and/or modify |
| 8 | # it under the terms of the GNU General Public License as published by |
| 9 | # the Free Software Foundation, version 3 of the License. |
| 10 | # |
| 11 | # septic is distributed in the hope that it will be useful, |
| 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 14 | # GNU General Public License for more details. |
| 15 | # |
| 16 | # You should have received a copy of the GNU General Public License |
| 17 | # along with septic. If not, see <http://www.gnu.org/licenses/>. |
| 18 | # |
| 19 | |
| 20 | # Base class for log handlers with common functionality |
| 21 | class ScmLogDbAdd |
| 22 | # Initialize log db add base class |
| 23 | def initialize(path, revision=nil) |
| 24 | # Get path |
| 25 | @path = normalize_repository_path(path) |
| 26 | # Cache email to user lookups |
| 27 | @email_to_user = { } |
| 28 | end |
| 29 | |
| 30 | protected |
| 31 | |
| 32 | # Get full path |
| 33 | def normalize_repository_path(path) |
| 34 | # Make sure git dir is without ending / |
| 35 | if path[-1..-1] == '/' |
| 36 | path = path[0..-2] |
| 37 | end |
| 38 | |
| 39 | # If path is ., expand to full path |
| 40 | path = Dir.pwd if path == '.' |
| 41 | |
| 42 | path |
| 43 | end |
| 44 | |
| 45 | # Parse author into author and email |
| 46 | def parse_author(author) |
| 47 | # Get email part from author if separated from the name |
| 48 | e_start = author.index('<') |
| 49 | e_end = author.rindex('>') |
| 50 | |
| 51 | # Found < >, assuming e-mail is inside |
| 52 | if e_start and e_end |
| 53 | email = author[e_start + 1,e_end - e_start - 1] |
| 54 | author = author[0,e_start].strip |
| 55 | else |
| 56 | email = author |
| 57 | author = '-' |
| 58 | end |
| 59 | |
| 60 | [author, email] |
| 61 | end |
| 62 | |
| 63 | # Lookup user id based on from email field |
| 64 | def get_user_from_email(email) |
| 65 | # Lookup, try to add if it does not exist |
| 66 | if not @email_to_user.include?(email) |
| 67 | @email_to_user[email] = User.find_by_email(email) |
| 68 | end |
| 69 | |
| 70 | @email_to_user[email] |
| 71 | end |
| 72 | end |