Revision 98f37c10
Von Sven Schöling vor fast 12 Jahren hinzugefügt
| SL/BackgroundJob/CsvImport.pm | ||
|---|---|---|
| 59 | 59 |
my ($self) = @_; |
| 60 | 60 |
|
| 61 | 61 |
my $c = SL::Controller::CsvImport->new; |
| 62 |
my $job = $self->{db_obj};
|
|
| 62 | 63 |
|
| 63 | 64 |
$c->profile($self->profile); |
| 64 |
$c->type($self->{db_obj}->data_as_hash->{type});
|
|
| 65 |
$c->type($job->data_as_hash->{type});
|
|
| 66 |
|
|
| 67 |
my $test = $job->data_as_hash->{test};
|
|
| 68 |
|
|
| 69 |
$self->track_progress( |
|
| 70 |
progress => 0, |
|
| 71 |
plan => {
|
|
| 72 |
'parsing csv' => 1, |
|
| 73 |
'building data' => 2, |
|
| 74 |
( 'saving data' => 3, )x!!$test, |
|
| 75 |
'building report' => ($test ? 3 : 4), |
|
| 76 |
}, |
|
| 77 |
num_phases => ($test ? 3 : 4), |
|
| 78 |
); |
|
| 65 | 79 |
$c->add_progress_tracker($self); |
| 66 | 80 |
|
| 67 |
$c->test_and_import(test => 1, session_id => $self->{db_obj}->data_as_hash->{session_id});
|
|
| 68 | 81 |
|
| 69 |
my $report_id = $c->save_report; |
|
| 70 |
$self->{db_obj}->set_data(report_id => $report_id);
|
|
| 71 |
$self->{db_obj}->save;
|
|
| 82 |
$c->test_and_import(test => 1, session_id => $job->data_as_hash->{session_id});
|
|
| 83 |
|
|
| 84 |
if ($c->errors) {
|
|
| 85 |
$job->set_data( |
|
| 86 |
errors => $c->errors, |
|
| 87 |
progress => -1, |
|
| 88 |
)->save; |
|
| 89 |
} else {
|
|
| 72 | 90 |
|
| 73 |
$c->track_progress(100); |
|
| 91 |
my $report_id = $c->save_report; |
|
| 92 |
$job->set_data(report_id => $report_id)->save; |
|
| 93 |
|
|
| 94 |
$c->track_progress(finished => 1); |
|
| 95 |
} |
|
| 74 | 96 |
} |
| 75 | 97 |
|
| 76 | 98 |
sub track_progress {
|
| 77 |
my ($self, $progress) = @_; |
|
| 99 |
my ($self, %params) = @_; |
|
| 100 |
|
|
| 101 |
my $data = $self->{db_obj}->data_as_hash;
|
|
| 102 |
my $progress = $data->{progress} || {};
|
|
| 78 | 103 |
|
| 104 |
$progress->{$_} = $params{$_} for keys %params;
|
|
| 79 | 105 |
$self->{db_obj}->set_data(progress => $progress);
|
| 80 | 106 |
$self->{db_obj}->save;
|
| 81 | 107 |
} |
| SL/Controller/CsvImport.pm | ||
|---|---|---|
| 89 | 89 |
|
| 90 | 90 |
$self->profile($profile); |
| 91 | 91 |
|
| 92 |
if ($data->{progress} < 100) {
|
|
| 92 |
if ($data->{errors} and my $first_error = $data->{errors}->[0]) {
|
|
| 93 |
flash('error', $::locale->text('There was an error parsing the csv file: #1 in line #2.', $first_error->[2], $first_error->[0]));
|
|
| 94 |
} |
|
| 95 |
|
|
| 96 |
if (!$data->{progress}{finished}) {
|
|
| 93 | 97 |
$self->render('csv_import/_deferred_results', { no_layout => 1 });
|
| 94 | 98 |
} else {
|
| 95 | 99 |
$self->action_report(report_id => $data->{report_id}, no_layout => 1);
|
| ... | ... | |
| 206 | 210 |
file => $self->csv_file_name, |
| 207 | 211 |
profile => $self->profile, |
| 208 | 212 |
type => $self->profile->type, |
| 213 |
test => $params{test},
|
|
| 209 | 214 |
)->save; |
| 210 | 215 |
|
| 211 | 216 |
SL::System::TaskServer->start_if_not_running; |
| ... | ... | |
| 234 | 239 |
|
| 235 | 240 |
$worker->run; |
| 236 | 241 |
|
| 242 |
return if $self->errors; |
|
| 243 |
|
|
| 237 | 244 |
$self->num_imported(0); |
| 238 | 245 |
$worker->save_objects if !$params{test};
|
| 239 | 246 |
|
| ... | ... | |
| 313 | 320 |
sub save_report {
|
| 314 | 321 |
my ($self, $report_id) = @_; |
| 315 | 322 |
|
| 323 |
$self->track_progress(phase => 'building report', progress => 0); |
|
| 324 |
|
|
| 316 | 325 |
my $clone_profile = $self->profile->clone_and_reset_deep; |
| 317 | 326 |
$clone_profile->save; # weird bug. if this isn't saved before adding it to the report, it will default back to the last profile. |
| 318 | 327 |
|
| ... | ... | |
| 349 | 358 |
my $o2 = $o1 + @methods; |
| 350 | 359 |
|
| 351 | 360 |
for my $row (0 .. $#{ $self->data }) {
|
| 361 |
$self->track_progress(progress => $row / @{ $self->data } * 100) if $row % 100 == 0;
|
|
| 352 | 362 |
my $data_row = $self->{data}[$row];
|
| 353 | 363 |
|
| 354 | 364 |
$sth->execute($report->id, $_, $row + 1, $data_row->{info_data}{ $info_methods[$_] }) for 0 .. $#info_methods;
|
| ... | ... | |
| 393 | 403 |
} |
| 394 | 404 |
|
| 395 | 405 |
sub track_progress {
|
| 396 |
my ($self, $progress) = @_;
|
|
| 406 |
my ($self, %params) = @_;
|
|
| 397 | 407 |
|
| 398 | 408 |
for my $tracker ($self->progress_tracker) {
|
| 399 |
$tracker->track_progress($progress);
|
|
| 409 |
$tracker->track_progress(%params);
|
|
| 400 | 410 |
} |
| 401 | 411 |
} |
| 402 | 412 |
|
| SL/Controller/CsvImport/Base.pm | ||
|---|---|---|
| 22 | 22 |
sub run {
|
| 23 | 23 |
my ($self) = @_; |
| 24 | 24 |
|
| 25 |
$self->controller->track_progress(0); |
|
| 25 |
$self->controller->track_progress(phase => 'parsing csv', progress => 0);
|
|
| 26 | 26 |
|
| 27 | 27 |
my $profile = $self->profile; |
| 28 | 28 |
$self->csv(SL::Helper::Csv->new(file => $self->file->file_name, |
| ... | ... | |
| 35 | 35 |
map { ( $_ => $self->controller->profile->get($_) ) } qw(sep_char escape_char quote_char),
|
| 36 | 36 |
)); |
| 37 | 37 |
|
| 38 |
$self->controller->track_progress(1);
|
|
| 38 |
$self->controller->track_progress(progress => 10);
|
|
| 39 | 39 |
|
| 40 | 40 |
my $old_numberformat = $::myconfig{numberformat};
|
| 41 | 41 |
$::myconfig{numberformat} = $self->controller->profile->get('numberformat');
|
| 42 | 42 |
|
| 43 | 43 |
$self->csv->parse; |
| 44 | 44 |
|
| 45 |
$self->controller->track_progress(3);
|
|
| 45 |
$self->controller->track_progress(progress => 50);
|
|
| 46 | 46 |
|
| 47 | 47 |
$self->controller->errors([ $self->csv->errors ]) if $self->csv->errors; |
| 48 | 48 |
|
| ... | ... | |
| 55 | 55 |
$self->controller->raw_data_headers({ used => { }, headers => [ ] });
|
| 56 | 56 |
$self->controller->info_headers({ used => { }, headers => [ ] });
|
| 57 | 57 |
|
| 58 |
$::lxdebug->dump(0, "self", $self->controller->info_headers); |
|
| 59 |
$::lxdebug->dump(0, "self", $self->controller->headers); |
|
| 60 |
$::lxdebug->dump(0, "self", $self->controller->raw_data_headers); |
|
| 61 |
|
|
| 62 | 58 |
my @objects = $self->csv->get_objects; |
| 63 | 59 |
|
| 64 |
$self->controller->track_progress(4);
|
|
| 60 |
$self->controller->track_progress(progress => 70);
|
|
| 65 | 61 |
|
| 66 | 62 |
my @raw_data = @{ $self->csv->get_data };
|
| 67 | 63 |
|
| 68 |
$self->controller->track_progress(4.5);
|
|
| 64 |
$self->controller->track_progress(progress => 80);
|
|
| 69 | 65 |
|
| 70 | 66 |
$self->controller->data([ pairwise { { object => $a, raw_data => $b, errors => [], information => [], info_data => {} } } @objects, @raw_data ]);
|
| 71 | 67 |
|
| 72 |
$self->controller->track_progress(5);
|
|
| 68 |
$self->controller->track_progress(progress => 90);
|
|
| 73 | 69 |
|
| 74 | 70 |
$self->check_objects; |
| 75 | 71 |
if ( $self->controller->profile->get('duplicates', 'no_check') ne 'no_check' ) {
|
| ... | ... | |
| 78 | 74 |
} |
| 79 | 75 |
$self->fix_field_lengths; |
| 80 | 76 |
|
| 81 |
$self->controller->track_progress(99);
|
|
| 77 |
$self->controller->track_progress(progress => 100);
|
|
| 82 | 78 |
|
| 83 | 79 |
$::myconfig{numberformat} = $old_numberformat;
|
| 84 | 80 |
} |
| ... | ... | |
| 387 | 383 |
return unless $data->[0]; |
| 388 | 384 |
return unless $data->[0]{object};
|
| 389 | 385 |
|
| 386 |
$self->controller->track_progress(phase => 'saving objects', progress => 0); # scale from 45..95%; |
|
| 387 |
|
|
| 390 | 388 |
my $dbh = $data->[0]{object}->db;
|
| 391 | 389 |
|
| 392 | 390 |
$dbh->begin_work; |
| ... | ... | |
| 404 | 402 |
} continue {
|
| 405 | 403 |
if ($entry_index % 100 == 0) {
|
| 406 | 404 |
$dbh->commit; |
| 407 |
$self->controller->track_progress(45 + $entry_index/scalar(@$data) * 50); # scale from 45..95%;
|
|
| 405 |
$self->controller->track_progress(progress => $entry_index/scalar(@$data) * 100); # scale from 45..95%;
|
|
| 408 | 406 |
$dbh->begin_work; |
| 409 | 407 |
} |
| 410 | 408 |
} |
| SL/Controller/CsvImport/Part.pm | ||
|---|---|---|
| 103 | 103 |
|
| 104 | 104 |
return unless @{ $self->controller->data };
|
| 105 | 105 |
|
| 106 |
$self->controller->track_progress(phase => 'building data', progress => 0); |
|
| 107 |
|
|
| 106 | 108 |
$self->makemodel_columns({});
|
| 107 | 109 |
|
| 108 | 110 |
my $i; |
| 109 | 111 |
my $num_data = scalar @{ $self->controller->data };
|
| 110 | 112 |
foreach my $entry (@{ $self->controller->data }) {
|
| 111 |
$self->controller->track_progress(8 + ($i/$num_data * 40)) if $i % 100 == 0; # scale from 5..45%
|
|
| 113 |
$self->controller->track_progress(progress => $i/$num_data * 100) if $i % 100 == 0;
|
|
| 112 | 114 |
|
| 113 | 115 |
$self->check_buchungsgruppe($entry); |
| 114 | 116 |
$self->check_type($entry); |
| SL/DB/BackgroundJob.pm | ||
|---|---|---|
| 83 | 83 |
my $data = YAML::Load($self->data); |
| 84 | 84 |
$data->{$_} = $data{$_} for keys %data;
|
| 85 | 85 |
$self->data(YAML::Dump($data)); |
| 86 |
|
|
| 87 |
$self; |
|
| 86 | 88 |
} |
| 87 | 89 |
|
| 88 | 90 |
sub validate {
|
| templates/webpages/csv_import/_deferred_results.html | ||
|---|---|---|
| 1 |
[%- USE L %] |
|
| 2 |
[%- USE T8 %] |
|
| 3 |
[%- USE HTML %] |
|
| 4 |
|
|
| 5 |
<h2>[% 'Import Status' | $T8 %]</h2> |
|
| 6 |
|
|
| 7 |
[% PROCESS 'common/flash.html' %] |
|
| 8 |
<div id='progress_description'></div> |
|
| 1 | 9 |
<div id='progressbar'></div> |
| 2 |
[% IF progress < 100 %] |
|
| 3 | 10 |
<script type='text/javascript'> |
| 4 | 11 |
function reload_results () {
|
| 5 | 12 |
$.ajax({
|
| ... | ... | |
| 13 | 20 |
error: function(e) { alert(e) },
|
| 14 | 21 |
}); |
| 15 | 22 |
} |
| 23 |
[%- UNLESS SELF.background_job.data_as_hash.progress < 0 %] |
|
| 16 | 24 |
$(document).ready(function(){
|
| 17 |
$('#progressbar').progressbar({ value: [% SELF.background_job.data_as_hash.progress * 1 %] });
|
|
| 25 |
$('#progress_description').html('[% SELF.background_job.data_as_hash.progress.plan.${SELF.background_job.data_as_hash.progress.phase} %] / [% SELF.background_job.data_as_hash.progress.num_phases %] [% SELF.background_job.data_as_hash.progress.phase | $T8 | html %]');
|
|
| 26 |
$('#progressbar').progressbar({ value: [% SELF.background_job.data_as_hash.progress.progress * 1 %] });
|
|
| 18 | 27 |
window.setTimeout(reload_results, 100); |
| 19 | 28 |
}) |
| 29 |
[%- END %] |
|
| 20 | 30 |
</script> |
| 21 |
[% END %] |
|
Auch abrufbar als: Unified diff
Progress genauer anzeigen