Revision d7492165
Von Sven Schöling vor fast 12 Jahren hinzugefügt
SL/BackgroundJob/CsvImport.pm | ||
---|---|---|
5 | 5 |
use parent qw(SL::BackgroundJob::Base); |
6 | 6 |
|
7 | 7 |
use YAML (); |
8 |
use SL::Controller::CsvImport; |
|
8 | 9 |
use SL::DB::CsvImportProfile; |
9 | 10 |
use SL::SessionFile::Random; |
10 | 11 |
|
... | ... | |
15 | 16 |
$package =~ s/SL::BackgroundJob:://; |
16 | 17 |
|
17 | 18 |
my $profile = delete $params{profile} || SL::DB::CsvImportProfile->new; |
18 |
my $result = delete $params{result} || SL::SessionFile::Random->new; |
|
19 |
my $new_profile = $profile->clone_and_reset_deep; |
|
20 |
$new_profile->save; |
|
19 | 21 |
|
20 | 22 |
my %data = ( |
21 |
profile => { $profile->flatten }, |
|
22 |
result => $result->file_name, |
|
23 | 23 |
%params, |
24 |
profile_id => $new_profile->id, |
|
25 |
session_id => $::auth->get_session_id, |
|
24 | 26 |
); |
25 | 27 |
|
26 | 28 |
my $job = SL::DB::BackgroundJob->new( |
... | ... | |
34 | 36 |
} |
35 | 37 |
|
36 | 38 |
sub profile { |
37 |
my ($self, $db_obj) = @_;
|
|
39 |
my ($self) = @_; |
|
38 | 40 |
|
39 | 41 |
if (!$self->{profile}) { |
40 |
$self->{profile} = SL::DB::CsvImportProfile->new; |
|
41 |
my $data = YAML::Load($db_obj->data); |
|
42 |
for (keys %$data) { |
|
43 |
$self->{profile}->set($_ => $data->{$_}); |
|
44 |
} |
|
42 |
my $data = YAML::Load($self->{db_obj}->data); |
|
43 |
$self->{profile} = SL::DB::Manager::CsvImportProfile->find_by(id => $data->{profile_id}); |
|
45 | 44 |
} |
46 | 45 |
|
47 | 46 |
return $self->{profile}; |
... | ... | |
60 | 59 |
my ($self) = @_; |
61 | 60 |
|
62 | 61 |
my $c = SL::Controller::CsvImport->new; |
63 |
$c->profile($self->{profile}); |
|
64 |
$c->test_and_import(test => $self->); |
|
65 | 62 |
|
63 |
$c->profile($self->profile); |
|
64 |
$c->type($self->{db_obj}->data_as_hash->{type}); |
|
65 |
$c->add_progress_tracker($self); |
|
66 |
|
|
67 |
$c->test_and_import(test => 1, session_id => $self->{db_obj}->data_as_hash->{session_id}); |
|
68 |
|
|
69 |
my $report_id = $c->save_report; |
|
70 |
$self->{db_obj}->set_data(report_id => $report_id); |
|
71 |
$self->{db_obj}->save; |
|
72 |
|
|
73 |
$c->track_progress(100); |
|
74 |
} |
|
75 |
|
|
76 |
sub track_progress { |
|
77 |
my ($self, $progress) = @_; |
|
66 | 78 |
|
79 |
$self->{db_obj}->set_data(progress => $progress); |
|
80 |
$self->{db_obj}->save; |
|
67 | 81 |
} |
68 | 82 |
|
69 | 83 |
sub cleanup { |
SL/Controller/CsvImport.pm | ||
---|---|---|
24 | 24 |
( |
25 | 25 |
scalar => [ qw(type profile file all_profiles all_charsets sep_char all_sep_chars quote_char all_quote_chars escape_char all_escape_chars all_buchungsgruppen all_units |
26 | 26 |
import_status errors headers raw_data_headers info_headers data num_imported num_importable displayable_columns file) ], |
27 |
'scalar --get_set_init' => [ qw(worker) ] |
|
27 |
'scalar --get_set_init' => [ qw(worker) ], |
|
28 |
'array' => [ |
|
29 |
progress_tracker => { }, |
|
30 |
add_progress_tracker => { interface => 'add', hash_key => 'progress_tracker' }, |
|
31 |
], |
|
28 | 32 |
); |
29 | 33 |
|
30 | 34 |
__PACKAGE__->run_before('check_auth'); |
... | ... | |
81 | 85 |
|
82 | 86 |
my $data = $self->{background_job}->data_as_hash; |
83 | 87 |
|
84 |
my $profile = SL::DB::CsvImportProfile->new(type => $data->{type}); |
|
85 |
my $profile_data = $data->{profile}; |
|
86 |
for (keys %$profile_data) { |
|
87 |
$profile->set($_ => $profile_data->{$_}); |
|
88 |
} |
|
88 |
my $profile = SL::DB::Manager::CsvImportProfile->find_by(id => $data->{profile_id}); |
|
89 | 89 |
|
90 | 90 |
$self->profile($profile); |
91 | 91 |
|
92 | 92 |
if ($data->{progress} < 100) { |
93 | 93 |
$self->render('csv_import/_deferred_results', { no_layout => 1 }); |
94 | 94 |
} else { |
95 |
die 'what? done? panic, no idea what to do';
|
|
95 |
$self->action_report(report_id => $data->{report_id}, no_layout => 1);
|
|
96 | 96 |
} |
97 | 97 |
} |
98 | 98 |
|
... | ... | |
221 | 221 |
sub test_and_import { |
222 | 222 |
my ($self, %params) = @_; |
223 | 223 |
|
224 |
$self->profile_from_form; |
|
225 |
|
|
226 |
if ($::form->{file}) { |
|
227 |
my $file = SL::SessionFile->new($self->csv_file_name, mode => '>'); |
|
228 |
$file->fh->print($::form->{file}); |
|
229 |
$file->fh->close; |
|
230 |
} |
|
231 |
|
|
232 |
my $file = SL::SessionFile->new($self->csv_file_name, mode => '<', encoding => $self->profile->get('charset')); |
|
233 |
if (!$file->fh) { |
|
234 |
flash('error', $::locale->text('No file has been uploaded yet.')); |
|
235 |
return $self->action_new; |
|
236 |
} |
|
224 |
my $file = SL::SessionFile->new( |
|
225 |
$self->csv_file_name, |
|
226 |
mode => '<', |
|
227 |
encoding => $self->profile->get('charset'), |
|
228 |
session_id => $params{session_id} |
|
229 |
); |
|
237 | 230 |
|
238 | 231 |
$self->file($file); |
239 | 232 |
|
... | ... | |
244 | 237 |
$self->num_imported(0); |
245 | 238 |
$worker->save_objects if !$params{test}; |
246 | 239 |
|
247 |
$self->save_report; |
|
248 |
|
|
249 | 240 |
$self->num_importable(scalar grep { !$_ } map { scalar @{ $_->{errors} } } @{ $self->data || [] }); |
250 | 241 |
$self->import_status($params{test} ? 'tested' : 'imported'); |
251 | 242 |
|
252 |
flash('info', $::locale->text('Objects have been imported.')) if !$params{test}; |
|
243 |
# flash('info', $::locale->text('Objects have been imported.')) if !$params{test};
|
|
253 | 244 |
} |
254 | 245 |
|
255 | 246 |
sub load_default_profile { |
... | ... | |
401 | 392 |
$self->worker->setup_displayable_columns; |
402 | 393 |
} |
403 | 394 |
|
395 |
sub track_progress { |
|
396 |
my ($self, $progress) = @_; |
|
397 |
|
|
398 |
for my $tracker ($self->progress_tracker) { |
|
399 |
$tracker->track_progress($progress); |
|
400 |
} |
|
401 |
} |
|
402 |
|
|
404 | 403 |
|
405 | 404 |
1; |
SL/Controller/CsvImport/Base.pm | ||
---|---|---|
22 | 22 |
sub run { |
23 | 23 |
my ($self) = @_; |
24 | 24 |
|
25 |
$self->controller->track_progress(0); |
|
26 |
|
|
25 | 27 |
my $profile = $self->profile; |
26 | 28 |
$self->csv(SL::Helper::Csv->new(file => $self->file->file_name, |
27 | 29 |
encoding => $self->controller->profile->get('charset'), |
... | ... | |
33 | 35 |
map { ( $_ => $self->controller->profile->get($_) ) } qw(sep_char escape_char quote_char), |
34 | 36 |
)); |
35 | 37 |
|
38 |
$self->controller->track_progress(1); |
|
39 |
|
|
36 | 40 |
my $old_numberformat = $::myconfig{numberformat}; |
37 | 41 |
$::myconfig{numberformat} = $self->controller->profile->get('numberformat'); |
38 | 42 |
|
39 | 43 |
$self->csv->parse; |
40 | 44 |
|
45 |
$self->controller->track_progress(3); |
|
46 |
|
|
41 | 47 |
$self->controller->errors([ $self->csv->errors ]) if $self->csv->errors; |
42 | 48 |
|
43 | 49 |
return if ( !$self->csv->header || $self->csv->errors ); |
... | ... | |
49 | 55 |
$self->controller->raw_data_headers({ used => { }, headers => [ ] }); |
50 | 56 |
$self->controller->info_headers({ used => { }, headers => [ ] }); |
51 | 57 |
|
58 |
$::lxdebug->dump(0, "self", $self->controller->info_headers); |
|
59 |
$::lxdebug->dump(0, "self", $self->controller->headers); |
|
60 |
$::lxdebug->dump(0, "self", $self->controller->raw_data_headers); |
|
61 |
|
|
52 | 62 |
my @objects = $self->csv->get_objects; |
63 |
|
|
64 |
$self->controller->track_progress(4); |
|
65 |
|
|
53 | 66 |
my @raw_data = @{ $self->csv->get_data }; |
67 |
|
|
68 |
$self->controller->track_progress(4.5); |
|
69 |
|
|
54 | 70 |
$self->controller->data([ pairwise { { object => $a, raw_data => $b, errors => [], information => [], info_data => {} } } @objects, @raw_data ]); |
55 | 71 |
|
72 |
$self->controller->track_progress(5); |
|
73 |
|
|
56 | 74 |
$self->check_objects; |
57 | 75 |
if ( $self->controller->profile->get('duplicates', 'no_check') ne 'no_check' ) { |
58 | 76 |
$self->check_std_duplicates(); |
... | ... | |
60 | 78 |
} |
61 | 79 |
$self->fix_field_lengths; |
62 | 80 |
|
81 |
$self->controller->track_progress(99); |
|
82 |
|
|
63 | 83 |
$::myconfig{numberformat} = $old_numberformat; |
64 | 84 |
} |
65 | 85 |
|
... | ... | |
364 | 384 |
|
365 | 385 |
my $data = $params{data} || $self->controller->data; |
366 | 386 |
|
367 |
foreach my $entry (@{ $data }) { |
|
387 |
return unless $data->[0]; |
|
388 |
return unless $data->[0]{object}; |
|
389 |
|
|
390 |
my $dbh = $data->[0]{object}->db; |
|
391 |
|
|
392 |
$dbh->begin_work; |
|
393 |
foreach my $entry_index (0 .. $#$data) { |
|
394 |
my $entry = $data->[$entry_index]; |
|
368 | 395 |
next if @{ $entry->{errors} }; |
369 | 396 |
|
370 | 397 |
my $object = $entry->{object_to_save} || $entry->{object}; |
... | ... | |
374 | 401 |
} else { |
375 | 402 |
$self->controller->num_imported($self->controller->num_imported + 1); |
376 | 403 |
} |
404 |
} continue { |
|
405 |
if ($entry_index % 100 == 0) { |
|
406 |
$dbh->commit; |
|
407 |
$self->controller->track_progress(45 + $entry_index/scalar(@$data) * 50); # scale from 45..95%; |
|
408 |
$dbh->begin_work; |
|
409 |
} |
|
377 | 410 |
} |
411 |
$dbh->commit; |
|
378 | 412 |
} |
379 | 413 |
|
380 | 414 |
sub field_lengths { |
SL/Controller/CsvImport/Part.pm | ||
---|---|---|
105 | 105 |
|
106 | 106 |
$self->makemodel_columns({}); |
107 | 107 |
|
108 |
my $i; |
|
109 |
my $num_data = scalar @{ $self->controller->data }; |
|
108 | 110 |
foreach my $entry (@{ $self->controller->data }) { |
111 |
$self->controller->track_progress(8 + ($i/$num_data * 40)) if $i % 100 == 0; # scale from 5..45% |
|
112 |
|
|
109 | 113 |
$self->check_buchungsgruppe($entry); |
110 | 114 |
$self->check_type($entry); |
111 | 115 |
$self->check_unit($entry); |
... | ... | |
120 | 124 |
$self->handle_cvars($entry); |
121 | 125 |
$self->handle_makemodel($entry); |
122 | 126 |
$self->set_various_fields($entry); |
127 |
} continue { |
|
128 |
$i++; |
|
123 | 129 |
} |
124 | 130 |
|
125 | 131 |
$self->add_columns(qw(type)) if $self->settings->{parts_type} eq 'mixed'; |
... | ... | |
496 | 502 |
} |
497 | 503 |
} |
498 | 504 |
|
499 |
1; |
|
505 |
1; |
Auch abrufbar als: Unified diff
Weitere Verbesserungen am asynchronen Import.
- tracking
- profile/session handling