jacknudt
/
socialforge
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

优化zip重复打包问题

This commit is contained in:
guange 2015-04-02 17:20:52 +08:00
parent 0118b10725
commit 56df5ce9a7
10 changed files with 157 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Gemfile
View File

@ -4,10 +4,9 @@ source 'http://ruby.taobao.org'
unless RUBY_PLATFORM =~ /w32/
# unix-like only
gem 'iconv'
gem 'rubyzip'
gem 'zip-zip'
end
gem 'zipruby', '~> 0.3.6' #performance
gem 'delayed_job_active_record'#, :group => :production
gem 'daemons'
gem 'grape', '~> 0.9.0'

130
app/controllers/zipdown_controller.rb
View File

@ -1,4 +1,3 @@
require 'zip'
class ZipdownController < ApplicationController
#查找项目（课程）
before_filter :find_project_by_bid_id, :only => [:assort]
@ -56,9 +55,9 @@ class ZipdownController < ApplicationController
if homework != nil
unless homework.attachments.empty?
zipfile = zip_homework_by_user homework
send_file zipfile, :filename => ((homework.user.user_extensions.nil? || homework.user.user_extensions.student_id.nil?) ? "" : homework.user.user_extensions.student_id) +
send_file zipfile.file_path, :filename => ((homework.user.user_extensions.nil? || homework.user.user_extensions.student_id.nil?) ? "" : homework.user.user_extensions.student_id) +
"_" + (homework.user.lastname.nil? ? "" : homework.user.lastname) + (homework.user.firstname.nil? ? "" : homework.user.firstname) +
"_" + homework.name + ".zip", :type => detect_content_type(zipfile) if(zipfile)
"_" + homework.name + ".zip", :type => detect_content_type(zipfile.file_path) if(zipfile)
else
render file: 'public/no_file_found.html'
end
@ -88,85 +87,120 @@ class ZipdownController < ApplicationController
def zip_bid(bid)
# Todo: User Access Controll
bid_homework_path = []
digests = []
bid.homeworks.each do |homeattach|
unless homeattach.attachments.empty?
bid_homework_path << zip_homework_by_user(homeattach)
out_file = zip_homework_by_user(homeattach)
bid_homework_path << out_file.file_path
digests << out_file.file_digest
end
end
zips = split_pack_files(bid_homework_path, Setting.pack_attachment_max_size.to_i*1024)
x = 0
homework_id = bid.id
user_id = bid.author_id
zips.each { |o|
x += 1
file = zipping "#{Time.now.to_i}_#{bid.name}_#{x}.zip", o[:files], OUTPUT_FOLDER
o[:real_file] = file
o[:file] = File.basename(file)
o[:size] = (File.size(file) / 1024.0 / 1024.0).round(2)
out_file = find_or_pack(homework_id, user_id, digests.sort){
zipping("#{Time.now.to_i}_#{bid.name}.zip",
bid_homework_path, OUTPUT_FOLDER)
}
# zips = split_pack_files(bid_homework_path, Setting.pack_attachment_max_size.to_i*1024)
# x = 0
#
#
# zips.each { |o|
# x += 1
# file = zipping "#{Time.now.to_i}_#{bid.name}_#{x}.zip", o[:files], OUTPUT_FOLDER
# o[:real_file] = file
# o[:file] = File.basename(file)
# o[:size] = (File.size(file) / 1024.0 / 1024.0).round(2)
# }
[{files:[out_file.file_path], count: 1, index: 1,
real_file: out_file.file_path, file: File.basename(out_file.file_path),
size:(out_file.pack_size / 1024.0 / 1024.0).round(2)
}]
end
def zip_homework_by_user(homework_attach)
homeworks_attach_path = []
not_exist_file = []
# 需要将所有homework.attachments遍历加入zip
digests = []
homework_attach.attachments.each do |attach|
if File.exist?(attach.diskfile)
homeworks_attach_path << attach.diskfile
digests << attach.digest
else
not_exist_file << attach.filename
digests << 'not_exist_file'
end
end
out_file = find_or_pack(homework_attach.bid_id, homework_attach.user_id, digests.sort){
zipping("#{homework_attach.user.lastname}#{homework_attach.user.firstname}_#{((homework_attach.user.user_extensions.nil? || homework_attach.user.user_extensions.student_id.nil?) ? "" : homework_attach.user.user_extensions.student_id)}_#{Time.now.to_i.to_s}.zip",
homeworks_attach_path, OUTPUT_FOLDER, true, not_exist_file)
}
end
def zip_homework_by_user(homeattach)
homeworks_attach_path = []
not_exist_file = []
# 需要将所有homework.attachments遍历加入zip
# 并且返回zip路径
homeattach.attachments.each do |attach|
if File.exist?(attach.diskfile)
homeworks_attach_path << attach.diskfile
else
not_exist_file << attach.filename
end
def find_or_pack(homework_id, user_id, digests)
raise "please given a pack block" unless block_given?
out_file = ZipPack.packed?(homework_id, user_id, digests.sort)
unless out_file && out_file.file_valid?
file = yield
ZipPack.where(homework_id: homework_id,
user_id: user_id).delete_all
out_file = ZipPack.create(homework_id: homework_id,
user_id: user_id,
file_digest: Trustie::Utils.digest(file),
file_path: file,
pack_size: File.size(file),
file_digests: digests.join(',')
)
else
out_file.pack_times = out_file.pack_times + 1
out_file.save
end
zipping("#{homeattach.user.lastname}#{homeattach.user.firstname}_#{((homeattach.user.user_extensions.nil? || homeattach.user.user_extensions.student_id.nil?) ? "" : homeattach.user.user_extensions.student_id)}_#{Time.now.to_i.to_s}.zip", homeworks_attach_path, OUTPUT_FOLDER, true, not_exist_file)
out_file
end
def zipping(zip_name_refer, files_paths, output_path, is_attachment=false, not_exist_file=[])
# 输入待打包的文件列表，已经打包文件定位到ouput_path
ic = Iconv.new('GBK//IGNORE', 'UTF-8//IGNORE')
rename_zipfile = zip_name_refer ||= "#{Time.now.to_i.to_s}.zip"
zipfile_name = "#{output_path}/#{rename_zipfile}"
Dir.mkdir(File.dirname(zipfile_name)) unless File.exist?(File.dirname(zipfile_name))
unless is_attachment
#都是zip合并，没必要再费力压缩了
Zip.default_compression = Zlib::NO_COMPRESSION
else
Zip.default_compression = Zlib::DEFAULT_COMPRESSION
end
Zip::File.open(zipfile_name, Zip::File::CREATE) do |zipfile|
Zip::Archive.open(zipfile_name, Zip::CREATE) do |zipfile|
files_paths.each do |filename|
flag = true
index = 1
rename_file = ic.iconv( (File.basename(filename)) ).to_s
rename_file = ic.iconv( filename_to_real( File.basename(filename))).to_s if is_attachment
rename_file = File.basename(filename)
rename_file = filename_to_real( File.basename(filename)) if is_attachment
begin
zipfile.add(rename_file, filename)
zipfile.add_file(rename_file, filename)
flag = false
rescue Exception => e
zipfile.get_output_stream('FILE_NOTICE.txt') do |os|
os.write l(:label_file_exist)
end
zipfile.add_buffer('FILE_NOTICE.txt', l(:label_file_exist))
next
end
end
unless not_exist_file.empty?
zipfile.get_output_stream('FILE_LOST.txt') do |os|
os.write l(:label_file_lost) + not_exist_file.join(',').to_s
end
zipfile.add_buffer('FILE_LOST.txt', l(:label_file_lost) + not_exist_file.join(',').to_s)
end
end
zipfile_name
#rescue Errno => e
# logger.error "[zipdown#zipping] ===> #{e}"
# @error = e
end
# 合理分配文件打包

18
app/models/zip_pack.rb Normal file
View File

@ -0,0 +1,18 @@
class ZipPack < ActiveRecord::Base
# attr_accessible :title, :body
def self.packed?(bid_id, user_id, digests)
zip_pack = ZipPack.where(homework_id: bid_id, user_id: user_id).first
return false unless zip_pack && zip_pack.digests == digests
zip_pack
end
def file_valid?
return false unless File.exist?(self.file_path)
Trustie::Utils.digest(self.file_path) == self.file_digest
end
def digests
self.file_digests.split(',').sort
end
end

4
app/views/bids/_homework_list.html.erb
View File

@ -22,8 +22,8 @@
<%= link_to "留言", get_homework_jours_homework_attach_index_path(:bid_id => @bid.id), {:remote => true}%>
(<span id="jours_count" class="c_red f_12"><%= @jours_count %></span>)
</li>
<li>
<%#= link_to "作品打包下载", zipdown_assort_path(obj_class: @bid.class, obj_id: @bid), class: "tb_all" unless @bid.homeworks.empty? %>
<li><%= link_to "作品打包下载", zipdown_assort_path(obj_class: @bid.class, obj_id: @bid, format: :json),
remote: true, class: "tb_all" unless @bid.homeworks.empty? %>
</li>
</ul>
<% else %>

1
config/initializers/30-redmine.rb
View File

@ -2,6 +2,7 @@ I18n.default_locale = 'en'
I18n.backend = Redmine::I18n::Backend.new
require 'redmine'
require 'trustie'
# Load the secret token from the Redmine configuration file
secret = Redmine::Configuration['secret_token']

10
config/routes.rb
View File

@ -102,11 +102,11 @@ RedmineApp::Application.routes.draw do
mount SeemsRateable::Engine => '/rateable', :as => :rateable
# namespace :zipdown do
# match 'assort'
# match 'download_user_homework', :as => :download_user_homework
# match 'download'
# end
namespace :zipdown do
match 'assort'
match 'download_user_homework', :as => :download_user_homework
match 'download'
end
namespace :test do
match 'courselist'
match 'zip'

14
db/migrate/20150402015402_create_zip_packs.rb Normal file
View File

@ -0,0 +1,14 @@
class CreateZipPacks < ActiveRecord::Migration
def change
create_table :zip_packs do |t|
t.integer :user_id
t.integer :homework_id
t.string :file_digest
t.string :file_path
t.integer :pack_times, default: 1
t.integer :pack_size, default: 0
t.string :file_digests
t.timestamps
end
end
end

14
db/schema.rb
View File

@ -11,7 +11,7 @@
#
# It's strongly recommended to check this file into your version control system.
ActiveRecord::Schema.define(:version => 20150331032810) do
ActiveRecord::Schema.define(:version => 20150402015402) do
create_table "activities", :force => true do |t|
t.integer "act_id", :null => false
@ -1451,4 +1451,16 @@ ActiveRecord::Schema.define(:version => 20150331032810) do
t.datetime "updated_at", :null => false
end
create_table "zip_packs", :force => true do |t|
t.integer "user_id"
t.integer "homework_id"
t.string "file_digest"
t.string "file_path"
t.integer "pack_times", :default => 1
t.integer "pack_size", :default => 0
t.string "file_digests"
t.datetime "created_at", :null => false
t.datetime "updated_at", :null => false
end
end

1
lib/trustie.rb Normal file
View File

@ -0,0 +1 @@
require 'trustie/utils'

20
lib/trustie/utils.rb Normal file
View File

@ -0,0 +1,20 @@
#coding=utf-8
module Trustie
module Utils
def self.digest(diskfile)
md5 = Digest::MD5.new
File.open(diskfile, "rb") do |f|
buffer = ""
while (buffer = f.read(8192))
md5.update(buffer)
end
end
md5.hexdigest
end
end
end
if __FILE__ == $0
puts Trustie::Utils.digest('/Users/guange/Downloads/QQ_V4.0.2.dmg')
end