Projekt

Allgemein

Profil

« Zurück | Weiter » 

Revision d7492165

Von Sven Schöling vor fast 12 Jahren hinzugefügt

  • ID d74921657d967900bded4aaf4805647e9cf83562
  • Vorgänger 93d4967b
  • Nachfolger 98f37c10

Weitere Verbesserungen am asynchronen Import.

- tracking
- profile/session handling

Unterschiede anzeigen:

SL/BackgroundJob/CsvImport.pm
5 5
use parent qw(SL::BackgroundJob::Base);
6 6

  
7 7
use YAML ();
8
use SL::Controller::CsvImport;
8 9
use SL::DB::CsvImportProfile;
9 10
use SL::SessionFile::Random;
10 11

  
......
15 16
  $package          =~ s/SL::BackgroundJob:://;
16 17

  
17 18
  my $profile = delete $params{profile} || SL::DB::CsvImportProfile->new;
18
  my $result  = delete $params{result}  || SL::SessionFile::Random->new;
19
  my $new_profile = $profile->clone_and_reset_deep;
20
  $new_profile->save;
19 21

  
20 22
  my %data = (
21
    profile => { $profile->flatten },
22
    result  => $result->file_name,
23 23
    %params,
24
    profile_id => $new_profile->id,
25
    session_id => $::auth->get_session_id,
24 26
  );
25 27

  
26 28
  my $job = SL::DB::BackgroundJob->new(
......
34 36
}
35 37

  
36 38
sub profile {
37
  my ($self, $db_obj) = @_;
39
  my ($self) = @_;
38 40

  
39 41
  if (!$self->{profile}) {
40
    $self->{profile} = SL::DB::CsvImportProfile->new;
41
    my $data = YAML::Load($db_obj->data);
42
    for (keys %$data) {
43
      $self->{profile}->set($_ => $data->{$_});
44
    }
42
    my $data = YAML::Load($self->{db_obj}->data);
43
    $self->{profile} = SL::DB::Manager::CsvImportProfile->find_by(id => $data->{profile_id});
45 44
  }
46 45

  
47 46
  return $self->{profile};
......
60 59
  my ($self) = @_;
61 60

  
62 61
  my $c = SL::Controller::CsvImport->new;
63
  $c->profile($self->{profile});
64
  $c->test_and_import(test => $self->);
65 62

  
63
  $c->profile($self->profile);
64
  $c->type($self->{db_obj}->data_as_hash->{type});
65
  $c->add_progress_tracker($self);
66

  
67
  $c->test_and_import(test => 1, session_id => $self->{db_obj}->data_as_hash->{session_id});
68

  
69
  my $report_id = $c->save_report;
70
  $self->{db_obj}->set_data(report_id => $report_id);
71
  $self->{db_obj}->save;
72

  
73
  $c->track_progress(100);
74
}
75

  
76
sub track_progress {
77
  my ($self, $progress) = @_;
66 78

  
79
  $self->{db_obj}->set_data(progress => $progress);
80
  $self->{db_obj}->save;
67 81
}
68 82

  
69 83
sub cleanup {
SL/Controller/CsvImport.pm
24 24
(
25 25
 scalar                  => [ qw(type profile file all_profiles all_charsets sep_char all_sep_chars quote_char all_quote_chars escape_char all_escape_chars all_buchungsgruppen all_units
26 26
                                 import_status errors headers raw_data_headers info_headers data num_imported num_importable displayable_columns file) ],
27
 'scalar --get_set_init' => [ qw(worker) ]
27
 'scalar --get_set_init' => [ qw(worker) ],
28
 'array'                 => [
29
   progress_tracker     => { },
30
   add_progress_tracker => {  interface => 'add', hash_key => 'progress_tracker' },
31
 ],
28 32
);
29 33

  
30 34
__PACKAGE__->run_before('check_auth');
......
81 85

  
82 86
  my $data = $self->{background_job}->data_as_hash;
83 87

  
84
  my $profile = SL::DB::CsvImportProfile->new(type => $data->{type});
85
  my $profile_data = $data->{profile};
86
  for (keys %$profile_data) {
87
    $profile->set($_ => $profile_data->{$_});
88
  }
88
  my $profile = SL::DB::Manager::CsvImportProfile->find_by(id => $data->{profile_id});
89 89

  
90 90
  $self->profile($profile);
91 91

  
92 92
  if ($data->{progress} < 100) {
93 93
    $self->render('csv_import/_deferred_results', { no_layout => 1 });
94 94
  } else {
95
   die 'what? done? panic, no idea what to do';
95
    $self->action_report(report_id => $data->{report_id}, no_layout => 1);
96 96
  }
97 97
}
98 98

  
......
221 221
sub test_and_import {
222 222
  my ($self, %params) = @_;
223 223

  
224
  $self->profile_from_form;
225

  
226
  if ($::form->{file}) {
227
    my $file = SL::SessionFile->new($self->csv_file_name, mode => '>');
228
    $file->fh->print($::form->{file});
229
    $file->fh->close;
230
  }
231

  
232
  my $file = SL::SessionFile->new($self->csv_file_name, mode => '<', encoding => $self->profile->get('charset'));
233
  if (!$file->fh) {
234
    flash('error', $::locale->text('No file has been uploaded yet.'));
235
    return $self->action_new;
236
  }
224
  my $file = SL::SessionFile->new(
225
    $self->csv_file_name,
226
    mode       => '<',
227
    encoding   => $self->profile->get('charset'),
228
    session_id => $params{session_id}
229
  );
237 230

  
238 231
  $self->file($file);
239 232

  
......
244 237
  $self->num_imported(0);
245 238
  $worker->save_objects if !$params{test};
246 239

  
247
  $self->save_report;
248

  
249 240
  $self->num_importable(scalar grep { !$_ } map { scalar @{ $_->{errors} } } @{ $self->data || [] });
250 241
  $self->import_status($params{test} ? 'tested' : 'imported');
251 242

  
252
  flash('info', $::locale->text('Objects have been imported.')) if !$params{test};
243
#  flash('info', $::locale->text('Objects have been imported.')) if !$params{test};
253 244
}
254 245

  
255 246
sub load_default_profile {
......
401 392
  $self->worker->setup_displayable_columns;
402 393
}
403 394

  
395
sub track_progress {
396
  my ($self, $progress) = @_;
397

  
398
  for my $tracker ($self->progress_tracker) {
399
    $tracker->track_progress($progress);
400
  }
401
}
402

  
404 403

  
405 404
1;
SL/Controller/CsvImport/Base.pm
22 22
sub run {
23 23
  my ($self) = @_;
24 24

  
25
  $self->controller->track_progress(0);
26

  
25 27
  my $profile = $self->profile;
26 28
  $self->csv(SL::Helper::Csv->new(file                   => $self->file->file_name,
27 29
                                  encoding               => $self->controller->profile->get('charset'),
......
33 35
                                  map { ( $_ => $self->controller->profile->get($_) ) } qw(sep_char escape_char quote_char),
34 36
                                 ));
35 37

  
38
  $self->controller->track_progress(1);
39

  
36 40
  my $old_numberformat      = $::myconfig{numberformat};
37 41
  $::myconfig{numberformat} = $self->controller->profile->get('numberformat');
38 42

  
39 43
  $self->csv->parse;
40 44

  
45
  $self->controller->track_progress(3);
46

  
41 47
  $self->controller->errors([ $self->csv->errors ]) if $self->csv->errors;
42 48

  
43 49
  return if ( !$self->csv->header || $self->csv->errors );
......
49 55
  $self->controller->raw_data_headers({ used => { }, headers => [ ] });
50 56
  $self->controller->info_headers({ used => { }, headers => [ ] });
51 57

  
58
  $::lxdebug->dump(0,  "self", $self->controller->info_headers);
59
  $::lxdebug->dump(0,  "self", $self->controller->headers);
60
  $::lxdebug->dump(0,  "self", $self->controller->raw_data_headers);
61

  
52 62
  my @objects  = $self->csv->get_objects;
63

  
64
  $self->controller->track_progress(4);
65

  
53 66
  my @raw_data = @{ $self->csv->get_data };
67

  
68
  $self->controller->track_progress(4.5);
69

  
54 70
  $self->controller->data([ pairwise { { object => $a, raw_data => $b, errors => [], information => [], info_data => {} } } @objects, @raw_data ]);
55 71

  
72
  $self->controller->track_progress(5);
73

  
56 74
  $self->check_objects;
57 75
  if ( $self->controller->profile->get('duplicates', 'no_check') ne 'no_check' ) {
58 76
    $self->check_std_duplicates();
......
60 78
  }
61 79
  $self->fix_field_lengths;
62 80

  
81
  $self->controller->track_progress(99);
82

  
63 83
  $::myconfig{numberformat} = $old_numberformat;
64 84
}
65 85

  
......
364 384

  
365 385
  my $data = $params{data} || $self->controller->data;
366 386

  
367
  foreach my $entry (@{ $data }) {
387
  return unless $data->[0];
388
  return unless $data->[0]{object};
389

  
390
  my $dbh = $data->[0]{object}->db;
391

  
392
  $dbh->begin_work;
393
  foreach my $entry_index (0 .. $#$data) {
394
    my $entry = $data->[$entry_index];
368 395
    next if @{ $entry->{errors} };
369 396

  
370 397
    my $object = $entry->{object_to_save} || $entry->{object};
......
374 401
    } else {
375 402
      $self->controller->num_imported($self->controller->num_imported + 1);
376 403
    }
404
  } continue {
405
    if ($entry_index % 100 == 0) {
406
      $dbh->commit;
407
      $self->controller->track_progress(45 + $entry_index/scalar(@$data) * 50); # scale from 45..95%;
408
      $dbh->begin_work;
409
    }
377 410
  }
411
  $dbh->commit;
378 412
}
379 413

  
380 414
sub field_lengths {
SL/Controller/CsvImport/Part.pm
105 105

  
106 106
  $self->makemodel_columns({});
107 107

  
108
  my $i;
109
  my $num_data = scalar @{ $self->controller->data };
108 110
  foreach my $entry (@{ $self->controller->data }) {
111
    $self->controller->track_progress(8 + ($i/$num_data * 40)) if $i % 100 == 0; # scale from 5..45%
112

  
109 113
    $self->check_buchungsgruppe($entry);
110 114
    $self->check_type($entry);
111 115
    $self->check_unit($entry);
......
120 124
    $self->handle_cvars($entry);
121 125
    $self->handle_makemodel($entry);
122 126
    $self->set_various_fields($entry);
127
  } continue {
128
    $i++;
123 129
  }
124 130

  
125 131
  $self->add_columns(qw(type)) if $self->settings->{parts_type} eq 'mixed';
......
496 502
  }
497 503
}
498 504

  
499
1;
505
1;

Auch abrufbar als: Unified diff