Line data Source code
1 : /**************************************************************************
2 : * Copyright(c) 1998-2007, ALICE Experiment at CERN, All rights reserved. *
3 : * *
4 : * Author: The ALICE Off-line Project. *
5 : * Contributors are mentioned in the code where appropriate. *
6 : * *
7 : * Permission to use, copy, modify and distribute this software and its *
8 : * documentation strictly for non-commercial purposes is hereby granted *
9 : * without fee, provided that the above copyright notice appears in all *
10 : * copies and that both the copyright notice and this permission notice *
11 : * appear in the supporting documentation. The authors make no claims *
12 : * about the suitability of this software for any purpose. It is *
13 : * provided "as is" without express or implied warranty. *
14 : **************************************************************************/
15 :
16 : // Author: Mihaela Gheata, 01/09/2008
17 :
18 : //==============================================================================
19 : // AliAnalysisAlien - AliEn utility class. Provides interface for creating
20 : // a personalized JDL, finding and creating a dataset.
21 : //==============================================================================
22 :
23 : #include "AliAnalysisAlien.h"
24 :
25 : #include "Riostream.h"
26 : #include "TEnv.h"
27 : #include "TKey.h"
28 : #include "TBits.h"
29 : #include "TError.h"
30 : #include "TROOT.h"
31 : #include "TSystem.h"
32 : #include "TInterpreter.h"
33 : #include "TFile.h"
34 : #include "TFileCollection.h"
35 : #include "TChain.h"
36 : #include "TObjString.h"
37 : #include "TObjArray.h"
38 : #include "TMacro.h"
39 : #include "TGrid.h"
40 : #include "TGridResult.h"
41 : #include "TGridCollection.h"
42 : #include "TGridJDL.h"
43 : #include "TGridJobStatusList.h"
44 : #include "TGridJobStatus.h"
45 : #include "TFileMerger.h"
46 : #include "AliAnalysisManager.h"
47 : #include "AliAnalysisTaskCfg.h"
48 : #include "AliVEventHandler.h"
49 : #include "AliAnalysisDataContainer.h"
50 : #include "AliMultiInputEventHandler.h"
51 :
52 : using std::ofstream;
53 : using std::ifstream;
54 : using std::ios;
55 : using std::endl;
56 170 : ClassImp(AliAnalysisAlien)
57 : #if 0
58 : ;
59 : #endif
60 :
61 : namespace {
62 : Bool_t copyLocal2Alien(const char* where, const char* loc, const char* rem)
63 : {
64 0 : TString sl(Form("file:%s", loc));
65 0 : TString sr(Form("alien://%s", rem));
66 0 : Bool_t ret = TFile::Cp(sl, sr);
67 0 : if (!ret) {
68 0 : Warning(where, "Failed to copy %s to %s", sl.Data(), sr.Data());
69 : }
70 : return ret;
71 0 : }
72 : }
73 :
74 : //______________________________________________________________________________
75 : AliAnalysisAlien::AliAnalysisAlien()
76 0 : :AliAnalysisGrid(),
77 0 : fGridJDL(NULL),
78 0 : fMergingJDL(NULL),
79 0 : fPrice(0),
80 0 : fTTL(0),
81 0 : fSplitMaxInputFileNumber(0),
82 0 : fMaxInitFailed(0),
83 0 : fMasterResubmitThreshold(0),
84 0 : fNtestFiles(0),
85 0 : fNrunsPerMaster(0),
86 0 : fMaxMergeFiles(0),
87 0 : fMaxMergeStages(0),
88 0 : fNsubmitted(0),
89 0 : fProductionMode(0),
90 0 : fOutputToRunNo(0),
91 0 : fMergeViaJDL(0),
92 0 : fFastReadOption(0),
93 0 : fOverwriteMode(1),
94 0 : fNreplicas(2),
95 0 : fNproofWorkers(0),
96 0 : fNproofWorkersPerSlave(0),
97 0 : fProofReset(0),
98 0 : fNMCevents(0),
99 0 : fNMCjobs(0),
100 0 : fRunNumbers(),
101 0 : fExecutable(),
102 0 : fExecutableCommand(),
103 0 : fArguments(),
104 0 : fExecutableArgs(),
105 0 : fAnalysisMacro(),
106 0 : fAnalysisSource(),
107 0 : fValidationScript(),
108 0 : fAdditionalRootLibs(),
109 0 : fAdditionalLibs(),
110 0 : fGeneratorLibs(),
111 0 : fSplitMode(),
112 0 : fAPIVersion(),
113 0 : fROOTVersion(),
114 0 : fAliROOTVersion(),
115 0 : fAliPhysicsVersion(),
116 0 : fExternalPackages(),
117 0 : fUser(),
118 0 : fGridWorkingDir(),
119 0 : fGridDataDir(),
120 0 : fDataPattern(),
121 0 : fGridOutputDir(),
122 0 : fOutputArchive(),
123 0 : fOutputFiles(),
124 0 : fInputFormat(),
125 0 : fDatasetName(),
126 0 : fJDLName(),
127 0 : fTerminateFiles(),
128 0 : fMergeExcludes(),
129 0 : fRegisterExcludes(),
130 0 : fIncludePath(),
131 0 : fCloseSE(),
132 0 : fFriendChainName(),
133 0 : fJobTag(),
134 0 : fOutputSingle(),
135 0 : fRunPrefix(),
136 0 : fProofCluster(),
137 0 : fProofDataSet(),
138 0 : fFileForTestMode(),
139 0 : fAliRootMode(),
140 0 : fProofProcessOpt(),
141 0 : fMergeDirName(),
142 0 : fInputFiles(0),
143 0 : fPackages(0),
144 0 : fModules(0),
145 0 : fProofParam(),
146 0 : fDropToShell(true),
147 0 : fMCLoop(false),
148 0 : fGridJobIDs(""),
149 0 : fGridStages(""),
150 0 : fFriendLibs(""),
151 0 : fTreeName()
152 0 : {
153 : // Dummy ctor.
154 0 : SetDefaults();
155 0 : }
156 :
157 : //______________________________________________________________________________
158 : AliAnalysisAlien::AliAnalysisAlien(const char *name)
159 0 : :AliAnalysisGrid(name),
160 0 : fGridJDL(NULL),
161 0 : fMergingJDL(NULL),
162 0 : fPrice(0),
163 0 : fTTL(0),
164 0 : fSplitMaxInputFileNumber(0),
165 0 : fMaxInitFailed(0),
166 0 : fMasterResubmitThreshold(0),
167 0 : fNtestFiles(0),
168 0 : fNrunsPerMaster(0),
169 0 : fMaxMergeFiles(0),
170 0 : fMaxMergeStages(0),
171 0 : fNsubmitted(0),
172 0 : fProductionMode(0),
173 0 : fOutputToRunNo(0),
174 0 : fMergeViaJDL(0),
175 0 : fFastReadOption(0),
176 0 : fOverwriteMode(1),
177 0 : fNreplicas(2),
178 0 : fNproofWorkers(0),
179 0 : fNproofWorkersPerSlave(0),
180 0 : fProofReset(0),
181 0 : fNMCevents(0),
182 0 : fNMCjobs(0),
183 0 : fRunNumbers(),
184 0 : fExecutable(),
185 0 : fExecutableCommand(),
186 0 : fArguments(),
187 0 : fExecutableArgs(),
188 0 : fAnalysisMacro(),
189 0 : fAnalysisSource(),
190 0 : fValidationScript(),
191 0 : fAdditionalRootLibs(),
192 0 : fAdditionalLibs(),
193 0 : fGeneratorLibs(),
194 0 : fSplitMode(),
195 0 : fAPIVersion(),
196 0 : fROOTVersion(),
197 0 : fAliROOTVersion(),
198 0 : fAliPhysicsVersion(),
199 0 : fExternalPackages(),
200 0 : fUser(),
201 0 : fGridWorkingDir(),
202 0 : fGridDataDir(),
203 0 : fDataPattern(),
204 0 : fGridOutputDir(),
205 0 : fOutputArchive(),
206 0 : fOutputFiles(),
207 0 : fInputFormat(),
208 0 : fDatasetName(),
209 0 : fJDLName(),
210 0 : fTerminateFiles(),
211 0 : fMergeExcludes(),
212 0 : fRegisterExcludes(),
213 0 : fIncludePath(),
214 0 : fCloseSE(),
215 0 : fFriendChainName(),
216 0 : fJobTag(),
217 0 : fOutputSingle(),
218 0 : fRunPrefix(),
219 0 : fProofCluster(),
220 0 : fProofDataSet(),
221 0 : fFileForTestMode(),
222 0 : fAliRootMode(),
223 0 : fProofProcessOpt(),
224 0 : fMergeDirName(),
225 0 : fInputFiles(0),
226 0 : fPackages(0),
227 0 : fModules(0),
228 0 : fProofParam(),
229 0 : fDropToShell(true),
230 0 : fMCLoop(false),
231 0 : fGridJobIDs(""),
232 0 : fGridStages(""),
233 0 : fFriendLibs(""),
234 0 : fTreeName()
235 0 : {
236 : // Default ctor.
237 0 : SetDefaults();
238 0 : }
239 :
240 : //______________________________________________________________________________
241 : AliAnalysisAlien::AliAnalysisAlien(const AliAnalysisAlien& other)
242 0 : :AliAnalysisGrid(other),
243 0 : fGridJDL(NULL),
244 0 : fMergingJDL(NULL),
245 0 : fPrice(other.fPrice),
246 0 : fTTL(other.fTTL),
247 0 : fSplitMaxInputFileNumber(other.fSplitMaxInputFileNumber),
248 0 : fMaxInitFailed(other.fMaxInitFailed),
249 0 : fMasterResubmitThreshold(other.fMasterResubmitThreshold),
250 0 : fNtestFiles(other.fNtestFiles),
251 0 : fNrunsPerMaster(other.fNrunsPerMaster),
252 0 : fMaxMergeFiles(other.fMaxMergeFiles),
253 0 : fMaxMergeStages(other.fMaxMergeStages),
254 0 : fNsubmitted(other.fNsubmitted),
255 0 : fProductionMode(other.fProductionMode),
256 0 : fOutputToRunNo(other.fOutputToRunNo),
257 0 : fMergeViaJDL(other.fMergeViaJDL),
258 0 : fFastReadOption(other.fFastReadOption),
259 0 : fOverwriteMode(other.fOverwriteMode),
260 0 : fNreplicas(other.fNreplicas),
261 0 : fNproofWorkers(other.fNproofWorkers),
262 0 : fNproofWorkersPerSlave(other.fNproofWorkersPerSlave),
263 0 : fProofReset(other.fProofReset),
264 0 : fNMCevents(other.fNMCevents),
265 0 : fNMCjobs(other.fNMCjobs),
266 0 : fRunNumbers(other.fRunNumbers),
267 0 : fExecutable(other.fExecutable),
268 0 : fExecutableCommand(other.fExecutableCommand),
269 0 : fArguments(other.fArguments),
270 0 : fExecutableArgs(other.fExecutableArgs),
271 0 : fAnalysisMacro(other.fAnalysisMacro),
272 0 : fAnalysisSource(other.fAnalysisSource),
273 0 : fValidationScript(other.fValidationScript),
274 0 : fAdditionalRootLibs(other.fAdditionalRootLibs),
275 0 : fAdditionalLibs(other.fAdditionalLibs),
276 0 : fGeneratorLibs(other.fGeneratorLibs),
277 0 : fSplitMode(other.fSplitMode),
278 0 : fAPIVersion(other.fAPIVersion),
279 0 : fROOTVersion(other.fROOTVersion),
280 0 : fAliROOTVersion(other.fAliROOTVersion),
281 0 : fAliPhysicsVersion(other.fAliPhysicsVersion),
282 0 : fExternalPackages(other.fExternalPackages),
283 0 : fUser(other.fUser),
284 0 : fGridWorkingDir(other.fGridWorkingDir),
285 0 : fGridDataDir(other.fGridDataDir),
286 0 : fDataPattern(other.fDataPattern),
287 0 : fGridOutputDir(other.fGridOutputDir),
288 0 : fOutputArchive(other.fOutputArchive),
289 0 : fOutputFiles(other.fOutputFiles),
290 0 : fInputFormat(other.fInputFormat),
291 0 : fDatasetName(other.fDatasetName),
292 0 : fJDLName(other.fJDLName),
293 0 : fTerminateFiles(other.fTerminateFiles),
294 0 : fMergeExcludes(other.fMergeExcludes),
295 0 : fRegisterExcludes(other.fRegisterExcludes),
296 0 : fIncludePath(other.fIncludePath),
297 0 : fCloseSE(other.fCloseSE),
298 0 : fFriendChainName(other.fFriendChainName),
299 0 : fJobTag(other.fJobTag),
300 0 : fOutputSingle(other.fOutputSingle),
301 0 : fRunPrefix(other.fRunPrefix),
302 0 : fProofCluster(other.fProofCluster),
303 0 : fProofDataSet(other.fProofDataSet),
304 0 : fFileForTestMode(other.fFileForTestMode),
305 0 : fAliRootMode(other.fAliRootMode),
306 0 : fProofProcessOpt(other.fProofProcessOpt),
307 0 : fMergeDirName(other.fMergeDirName),
308 0 : fInputFiles(0),
309 0 : fPackages(0),
310 0 : fModules(0),
311 0 : fProofParam(),
312 0 : fDropToShell(other.fDropToShell),
313 0 : fMCLoop(other.fMCLoop),
314 0 : fGridJobIDs(other.fGridJobIDs),
315 0 : fGridStages(other.fGridStages),
316 0 : fFriendLibs(other.fFriendLibs),
317 0 : fTreeName(other.fTreeName)
318 0 : {
319 : // Copy ctor.
320 0 : fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
321 0 : fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
322 0 : fRunRange[0] = other.fRunRange[0];
323 0 : fRunRange[1] = other.fRunRange[1];
324 0 : if (other.fInputFiles) {
325 0 : fInputFiles = new TObjArray();
326 0 : TIter next(other.fInputFiles);
327 : TObject *obj;
328 0 : while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
329 0 : fInputFiles->SetOwner();
330 0 : }
331 0 : if (other.fPackages) {
332 0 : fPackages = new TObjArray();
333 0 : TIter next(other.fPackages);
334 : TObject *obj;
335 0 : while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
336 0 : fPackages->SetOwner();
337 0 : }
338 0 : if (other.fModules) {
339 0 : fModules = new TObjArray();
340 0 : fModules->SetOwner();
341 0 : TIter next(other.fModules);
342 : AliAnalysisTaskCfg *mod, *crt;
343 0 : while ((crt=(AliAnalysisTaskCfg*)next())) {
344 0 : mod = new AliAnalysisTaskCfg(*crt);
345 0 : fModules->Add(mod);
346 : }
347 0 : }
348 0 : }
349 :
350 : //______________________________________________________________________________
351 : AliAnalysisAlien::~AliAnalysisAlien()
352 0 : {
353 : // Destructor.
354 0 : delete fGridJDL;
355 0 : delete fMergingJDL;
356 0 : delete fInputFiles;
357 0 : delete fPackages;
358 0 : delete fModules;
359 0 : fProofParam.DeleteAll();
360 0 : }
361 :
362 : //______________________________________________________________________________
363 : AliAnalysisAlien &AliAnalysisAlien::operator=(const AliAnalysisAlien& other)
364 : {
365 : // Assignment.
366 0 : if (this != &other) {
367 0 : AliAnalysisGrid::operator=(other);
368 0 : fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
369 0 : fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
370 0 : fPrice = other.fPrice;
371 0 : fTTL = other.fTTL;
372 0 : fSplitMaxInputFileNumber = other.fSplitMaxInputFileNumber;
373 0 : fMaxInitFailed = other.fMaxInitFailed;
374 0 : fMasterResubmitThreshold = other.fMasterResubmitThreshold;
375 0 : fNtestFiles = other.fNtestFiles;
376 0 : fNrunsPerMaster = other.fNrunsPerMaster;
377 0 : fMaxMergeFiles = other.fMaxMergeFiles;
378 0 : fMaxMergeStages = other.fMaxMergeStages;
379 0 : fNsubmitted = other.fNsubmitted;
380 0 : fProductionMode = other.fProductionMode;
381 0 : fOutputToRunNo = other.fOutputToRunNo;
382 0 : fMergeViaJDL = other.fMergeViaJDL;
383 0 : fFastReadOption = other.fFastReadOption;
384 0 : fOverwriteMode = other.fOverwriteMode;
385 0 : fNreplicas = other.fNreplicas;
386 0 : fNproofWorkers = other.fNproofWorkers;
387 0 : fNproofWorkersPerSlave = other.fNproofWorkersPerSlave;
388 0 : fProofReset = other.fProofReset;
389 0 : fNMCevents = other.fNMCevents;
390 0 : fNMCjobs = other.fNMCjobs;
391 0 : fRunNumbers = other.fRunNumbers;
392 0 : fExecutable = other.fExecutable;
393 0 : fExecutableCommand = other.fExecutableCommand;
394 0 : fArguments = other.fArguments;
395 0 : fExecutableArgs = other.fExecutableArgs;
396 0 : fAnalysisMacro = other.fAnalysisMacro;
397 0 : fAnalysisSource = other.fAnalysisSource;
398 0 : fValidationScript = other.fValidationScript;
399 0 : fAdditionalRootLibs = other.fAdditionalRootLibs;
400 0 : fAdditionalLibs = other.fAdditionalLibs;
401 0 : fGeneratorLibs = other.fGeneratorLibs;
402 0 : fSplitMode = other.fSplitMode;
403 0 : fAPIVersion = other.fAPIVersion;
404 0 : fROOTVersion = other.fROOTVersion;
405 0 : fAliROOTVersion = other.fAliROOTVersion;
406 0 : fAliPhysicsVersion = other.fAliPhysicsVersion;
407 0 : fExternalPackages = other.fExternalPackages;
408 0 : fUser = other.fUser;
409 0 : fGridWorkingDir = other.fGridWorkingDir;
410 0 : fGridDataDir = other.fGridDataDir;
411 0 : fDataPattern = other.fDataPattern;
412 0 : fGridOutputDir = other.fGridOutputDir;
413 0 : fOutputArchive = other.fOutputArchive;
414 0 : fOutputFiles = other.fOutputFiles;
415 0 : fInputFormat = other.fInputFormat;
416 0 : fDatasetName = other.fDatasetName;
417 0 : fJDLName = other.fJDLName;
418 0 : fTerminateFiles = other.fTerminateFiles;
419 0 : fMergeExcludes = other.fMergeExcludes;
420 0 : fRegisterExcludes = other.fRegisterExcludes;
421 0 : fIncludePath = other.fIncludePath;
422 0 : fCloseSE = other.fCloseSE;
423 0 : fFriendChainName = other.fFriendChainName;
424 0 : fJobTag = other.fJobTag;
425 0 : fOutputSingle = other.fOutputSingle;
426 0 : fRunPrefix = other.fRunPrefix;
427 0 : fProofCluster = other.fProofCluster;
428 0 : fProofDataSet = other.fProofDataSet;
429 0 : fFileForTestMode = other.fFileForTestMode;
430 0 : fAliRootMode = other.fAliRootMode;
431 0 : fProofProcessOpt = other.fProofProcessOpt;
432 0 : fMergeDirName = other.fMergeDirName;
433 0 : fDropToShell = other.fDropToShell;
434 0 : fMCLoop = other.fMCLoop;
435 0 : fGridJobIDs = other.fGridJobIDs;
436 0 : fGridStages = other.fGridStages;
437 0 : fFriendLibs = other.fFriendLibs;
438 0 : fTreeName = other.fTreeName;
439 0 : if (other.fInputFiles) {
440 0 : fInputFiles = new TObjArray();
441 0 : TIter next(other.fInputFiles);
442 : TObject *obj;
443 0 : while ((obj=next())) fInputFiles->Add(new TObjString(obj->GetName()));
444 0 : fInputFiles->SetOwner();
445 0 : }
446 0 : if (other.fPackages) {
447 0 : fPackages = new TObjArray();
448 0 : TIter next(other.fPackages);
449 : TObject *obj;
450 0 : while ((obj=next())) fPackages->Add(new TObjString(obj->GetName()));
451 0 : fPackages->SetOwner();
452 0 : }
453 0 : if (other.fModules) {
454 0 : fModules = new TObjArray();
455 0 : fModules->SetOwner();
456 0 : TIter next(other.fModules);
457 : AliAnalysisTaskCfg *mod, *crt;
458 0 : while ((crt=(AliAnalysisTaskCfg*)next())) {
459 0 : mod = new AliAnalysisTaskCfg(*crt);
460 0 : fModules->Add(mod);
461 : }
462 0 : }
463 : }
464 0 : return *this;
465 0 : }
466 :
467 : //______________________________________________________________________________
468 : void AliAnalysisAlien::AddAdditionalLibrary(const char *name)
469 : {
470 : // Add a single additional library to be loaded. Extension must be present.
471 0 : TString lib(name);
472 0 : if (!lib.Contains(".")) {
473 0 : Error("AddAdditionalLibrary", "Extension not defined for %s", name);
474 0 : return;
475 : }
476 0 : if (fAdditionalLibs.Contains(name)) {
477 0 : Warning("AddAdditionalLibrary", "Library %s already added.", name);
478 0 : return;
479 : }
480 0 : if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
481 0 : fAdditionalLibs += lib;
482 0 : }
483 :
484 : //______________________________________________________________________________
485 : void AliAnalysisAlien::AddModule(AliAnalysisTaskCfg *module)
486 : {
487 : // Adding a module. Checks if already existing. Becomes owned by this.
488 0 : if (!module) return;
489 0 : if (GetModule(module->GetName())) {
490 0 : Error("AddModule", "A module having the same name %s already added", module->GetName());
491 0 : return;
492 : }
493 0 : if (!fModules) {
494 0 : fModules = new TObjArray();
495 0 : fModules->SetOwner();
496 0 : }
497 0 : fModules->Add(module);
498 0 : }
499 :
500 : //______________________________________________________________________________
501 : void AliAnalysisAlien::AddModules(TObjArray *list)
502 : {
503 : // Adding a list of modules. Checks if already existing. Becomes owned by this.
504 0 : TIter next(list);
505 : AliAnalysisTaskCfg *module;
506 0 : while ((module = (AliAnalysisTaskCfg*)next())) AddModule(module);
507 0 : }
508 :
509 : //______________________________________________________________________________
510 : Bool_t AliAnalysisAlien::CheckDependencies()
511 : {
512 : // Check if all dependencies are satisfied. Reorder modules if needed.
513 0 : Int_t nmodules = GetNmodules();
514 0 : if (!nmodules) {
515 0 : Warning("CheckDependencies", "No modules added yet to check their dependencies");
516 0 : return kTRUE;
517 : }
518 : AliAnalysisTaskCfg *mod = 0;
519 : AliAnalysisTaskCfg *dep = 0;
520 0 : TString depname;
521 : Int_t i, j, k;
522 0 : for (i=0; i<nmodules; i++) {
523 0 : mod = (AliAnalysisTaskCfg*) fModules->At(i);
524 0 : Int_t ndeps = mod->GetNdeps();
525 : Int_t istart = i;
526 0 : for (j=0; j<ndeps; j++) {
527 0 : depname = mod->GetDependency(j);
528 0 : dep = GetModule(depname);
529 0 : if (!dep) {
530 0 : Error("CheckDependencies","Dependency %s not added for module %s",
531 0 : depname.Data(), mod->GetName());
532 0 : return kFALSE;
533 : }
534 0 : if (dep->NeedsDependency(mod->GetName())) {
535 0 : Error("CheckDependencies","Modules %s and %s circularly depend on each other",
536 0 : mod->GetName(), dep->GetName());
537 0 : return kFALSE;
538 : }
539 0 : Int_t idep = fModules->IndexOf(dep);
540 : // The dependency task must come first
541 0 : if (idep>i) {
542 : // Remove at idep and move all objects below up one slot
543 : // down to index i included.
544 0 : fModules->RemoveAt(idep);
545 0 : for (k=idep-1; k>=i; k--) fModules->AddAt(fModules->RemoveAt(k),k+1);
546 : istart = i;
547 0 : fModules->AddAt(dep, i++);
548 : }
549 : }
550 : //Redo from istart if dependencies were inserted
551 0 : if (i>istart) i=istart-1;
552 0 : }
553 0 : return kTRUE;
554 0 : }
555 :
556 : //______________________________________________________________________________
557 : AliAnalysisManager *AliAnalysisAlien::CreateAnalysisManager(const char *name, const char *filename)
558 : {
559 : // Create the analysis manager and optionally execute the macro in filename.
560 0 : AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
561 0 : if (mgr) {
562 0 : mgr->SetMCLoop(fMCLoop);
563 0 : return mgr;
564 : }
565 0 : mgr = new AliAnalysisManager(name);
566 0 : mgr->SetGridHandler((AliAnalysisGrid*)this);
567 0 : mgr->SetMCLoop(fMCLoop);
568 0 : if (strlen(filename)) {
569 0 : TString line = gSystem->ExpandPathName(filename);
570 0 : line.Prepend(".x ");
571 0 : gROOT->ProcessLine(line.Data());
572 0 : }
573 0 : return mgr;
574 0 : }
575 :
576 : //______________________________________________________________________________
577 : Int_t AliAnalysisAlien::GetNmodules() const
578 : {
579 : // Get number of modules.
580 0 : if (!fModules) return 0;
581 0 : return fModules->GetEntries();
582 0 : }
583 :
584 : //______________________________________________________________________________
585 : AliAnalysisTaskCfg *AliAnalysisAlien::GetModule(const char *name)
586 : {
587 : // Get a module by name.
588 0 : if (!fModules) return 0;
589 0 : return (AliAnalysisTaskCfg*)fModules->FindObject(name);
590 0 : }
591 :
592 : //______________________________________________________________________________
593 : Bool_t AliAnalysisAlien::LoadModule(AliAnalysisTaskCfg *mod)
594 : {
595 : // Load a given module.
596 0 : if (mod->IsLoaded()) return kTRUE;
597 0 : AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
598 0 : if (!mgr) {
599 0 : Error("LoadModule", "No analysis manager created yet. Use CreateAnalysisManager first.");
600 0 : return kFALSE;
601 : }
602 0 : Int_t ndeps = mod->GetNdeps();
603 0 : TString depname;
604 0 : for (Int_t j=0; j<ndeps; j++) {
605 0 : depname = mod->GetDependency(j);
606 0 : AliAnalysisTaskCfg *dep = GetModule(depname);
607 0 : if (!dep) {
608 0 : Error("LoadModule","Dependency %s not existing for module %s",
609 0 : depname.Data(), mod->GetName());
610 0 : return kFALSE;
611 : }
612 0 : if (!LoadModule(dep)) {
613 0 : Error("LoadModule","Dependency %s for module %s could not be loaded",
614 0 : depname.Data(), mod->GetName());
615 0 : return kFALSE;
616 : }
617 0 : }
618 : // Load libraries for the module
619 0 : if (!mod->CheckLoadLibraries()) {
620 0 : Error("LoadModule", "Cannot load all libraries for module %s", mod->GetName());
621 0 : return kFALSE;
622 : }
623 : // Check if a custom file name was requested
624 0 : if (strlen(mod->GetOutputFileName())) mgr->SetCommonFileName(mod->GetOutputFileName());
625 :
626 : // Check if a custom terminate file name was requested
627 0 : if (strlen(mod->GetTerminateFileName())) {
628 0 : if (!fTerminateFiles.IsNull()) fTerminateFiles += ",";
629 0 : fTerminateFiles += mod->GetTerminateFileName();
630 : }
631 :
632 : // Execute the macro
633 0 : if (mod->ExecuteMacro()<0) {
634 0 : Error("LoadModule", "Executing the macro %s with arguments: %s for module %s returned a negative value",
635 0 : mod->GetMacroName(), mod->GetMacroArgs(), mod->GetName());
636 0 : return kFALSE;
637 : }
638 : // Configure dependencies
639 0 : if (mod->GetConfigMacro() && mod->ExecuteConfigMacro()<0) {
640 0 : Error("LoadModule", "There was an error executing the deps config macro %s for module %s",
641 0 : mod->GetConfigMacro()->GetTitle(), mod->GetName());
642 0 : return kFALSE;
643 : }
644 : // Adjust extra libraries
645 0 : Int_t nlibs = mod->GetNlibs();
646 0 : TString lib;
647 0 : for (Int_t i=0; i<nlibs; i++) {
648 0 : lib = mod->GetLibrary(i);
649 0 : lib = Form("lib%s.so", lib.Data());
650 0 : if (fAdditionalLibs.Contains(lib)) continue;
651 0 : if (!fAdditionalLibs.IsNull()) fAdditionalLibs += " ";
652 0 : fAdditionalLibs += lib;
653 : }
654 : return kTRUE;
655 0 : }
656 :
657 : //______________________________________________________________________________
658 : Bool_t AliAnalysisAlien::GenerateTrain(const char *name)
659 : {
660 : // Generate the full train.
661 0 : fAdditionalLibs = "";
662 0 : if (!LoadModules()) return kFALSE;
663 0 : AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
664 0 : if (!mgr->InitAnalysis()) return kFALSE;
665 0 : mgr->RunLocalInit();
666 0 : mgr->PrintStatus();
667 0 : Int_t productionMode = fProductionMode;
668 0 : SetProductionMode();
669 0 : TString macro = fAnalysisMacro;
670 0 : TString executable = fExecutable;
671 0 : TString validation = fValidationScript;
672 0 : TString execCommand = fExecutableCommand;
673 0 : SetAnalysisMacro(Form("%s.C", name));
674 0 : SetExecutable(Form("%s.sh", name));
675 : // SetExecutableCommand("aliroot -b -q ");
676 0 : SetValidationScript(Form("%s_validation.sh", name));
677 0 : StartAnalysis();
678 0 : SetProductionMode(productionMode);
679 0 : fAnalysisMacro = macro;
680 0 : fExecutable = executable;
681 0 : fExecutableCommand = execCommand;
682 0 : fValidationScript = validation;
683 : return kTRUE;
684 0 : }
685 :
686 : //______________________________________________________________________________
687 : Bool_t AliAnalysisAlien::GenerateTest(const char *name, const char *modname)
688 : {
689 : // Generate test macros for a single module or for the full train.
690 0 : fAdditionalLibs = "";
691 0 : if (strlen(modname)) {
692 0 : if (!CheckDependencies()) return kFALSE;
693 0 : AliAnalysisTaskCfg *mod = GetModule(modname);
694 0 : if (!mod) {
695 0 : Error("GenerateTest", "cannot generate test for inexistent module %s", modname);
696 0 : return kFALSE;
697 : }
698 0 : if (!LoadModule(mod)) return kFALSE;
699 0 : if (!LoadFriendLibs()) return kFALSE;
700 0 : } else if (!LoadModules()) return kFALSE;
701 0 : AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
702 0 : if (!mgr->InitAnalysis()) return kFALSE;
703 0 : mgr->RunLocalInit();
704 0 : mgr->PrintStatus();
705 0 : SetLocalTest(kTRUE);
706 0 : Int_t productionMode = fProductionMode;
707 0 : SetProductionMode();
708 0 : TString macro = fAnalysisMacro;
709 0 : TString executable = fExecutable;
710 0 : TString validation = fValidationScript;
711 0 : TString execCommand = fExecutableCommand;
712 0 : SetAnalysisMacro(Form("%s.C", name));
713 0 : SetExecutable(Form("%s.sh", name));
714 0 : fOutputFiles = GetListOfFiles("outaod");
715 : // Add extra files registered to the analysis manager
716 0 : TString extra = GetListOfFiles("ext");
717 0 : if (!extra.IsNull()) {
718 0 : extra.ReplaceAll(".root", "*.root");
719 0 : if (!fOutputFiles.IsNull()) fOutputFiles += ",";
720 0 : fOutputFiles += extra;
721 : }
722 : // SetExecutableCommand("aliroot -b -q ");
723 0 : SetValidationScript(Form("%s_validation.sh", name));
724 0 : WriteAnalysisFile();
725 0 : WriteAnalysisMacro();
726 0 : WriteExecutable();
727 0 : WriteValidationScript();
728 0 : WriteMergingMacro();
729 0 : WriteMergeExecutable();
730 0 : WriteValidationScript(kTRUE);
731 0 : SetLocalTest(kFALSE);
732 0 : SetProductionMode(productionMode);
733 0 : fAnalysisMacro = macro;
734 0 : fExecutable = executable;
735 0 : fExecutableCommand = execCommand;
736 0 : fValidationScript = validation;
737 : return kTRUE;
738 0 : }
739 :
740 : //______________________________________________________________________________
741 : Bool_t AliAnalysisAlien::LoadModules()
742 : {
743 : // Load all modules by executing the AddTask macros. Checks first the dependencies.
744 0 : fAdditionalLibs = "";
745 0 : Int_t nmodules = GetNmodules();
746 0 : if (!nmodules) {
747 0 : Warning("LoadModules", "No module to be loaded");
748 0 : return kTRUE;
749 : }
750 0 : AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
751 0 : if (!mgr) {
752 0 : Error("LoadModules", "No analysis manager created yet. Use CreateAnalysisManager first.");
753 0 : return kFALSE;
754 : }
755 0 : if (!CheckDependencies()) return kFALSE;
756 0 : nmodules = GetNmodules();
757 : AliAnalysisTaskCfg *mod;
758 0 : for (Int_t imod=0; imod<nmodules; imod++) {
759 0 : mod = (AliAnalysisTaskCfg*)fModules->At(imod);
760 0 : if (!LoadModule(mod)) return kFALSE;
761 : }
762 : // Load additional friend libraries
763 0 : return LoadFriendLibs();
764 0 : }
765 :
766 : //______________________________________________________________________________
767 : Bool_t AliAnalysisAlien::LoadFriendLibs() const
768 : {
769 : // Load libraries required for reading friends.
770 0 : if (fFriendLibs.Length()) {
771 : TObjArray *list = 0;
772 0 : TString lib;
773 0 : if (fFriendLibs.Contains(",")) list = fFriendLibs.Tokenize(",");
774 0 : else list = fFriendLibs.Tokenize(" ");
775 0 : for (Int_t ilib=0; ilib<list->GetEntriesFast(); ilib++) {
776 0 : lib = list->At(ilib)->GetName();
777 0 : lib.ReplaceAll(".so","");
778 0 : lib.ReplaceAll(".dylib","");
779 0 : lib.ReplaceAll(" ","");
780 0 : if (lib.BeginsWith("lib")) lib.Remove(0, 3);
781 0 : lib.Prepend("lib");
782 0 : Int_t loaded = strlen(gSystem->GetLibraries(lib,"",kFALSE));
783 0 : if (!loaded) loaded = gSystem->Load(lib);
784 0 : if (loaded < 0) {
785 0 : Error("LoadFriendLibs", "Cannot load library for friends %s", lib.Data());
786 0 : return kFALSE;
787 : }
788 0 : }
789 0 : delete list;
790 0 : }
791 0 : return kTRUE;
792 0 : }
793 :
794 : //______________________________________________________________________________
795 : void AliAnalysisAlien::SetRunPrefix(const char *prefix)
796 : {
797 : // Set the run number format. Can be a prefix or a format like "%09d"
798 0 : fRunPrefix = prefix;
799 0 : if (!fRunPrefix.Contains("%")) fRunPrefix += "%d";
800 0 : }
801 :
802 : //______________________________________________________________________________
803 : void AliAnalysisAlien::AddIncludePath(const char *path)
804 : {
805 : // Add include path in the remote analysis macro.
806 0 : TString p(path);
807 0 : if (p.Contains("-I")) fIncludePath += Form("%s ", path);
808 0 : else fIncludePath += Form("-I%s ", path);
809 0 : }
810 :
811 : //______________________________________________________________________________
812 : void AliAnalysisAlien::AddRunNumber(Int_t run)
813 : {
814 : // Add a run number to the list of runs to be processed.
815 0 : if (fRunNumbers.Length()) fRunNumbers += " ";
816 0 : fRunNumbers += Form(fRunPrefix.Data(), run);
817 0 : }
818 :
819 : //______________________________________________________________________________
820 : void AliAnalysisAlien::AddRunList(const char* runList)
821 : {
822 : // Add several runs into the list of runs; they are expected to be separated by a blank character.
823 0 : TString sList = runList;
824 0 : TObjArray *list = sList.Tokenize(" ");
825 0 : Int_t n = list->GetEntries();
826 0 : for (Int_t i = 0; i < n; i++) {
827 0 : TObjString *os = (TObjString*)list->At(i);
828 0 : AddRunNumber(os->GetString().Atoi());
829 : }
830 0 : delete list;
831 0 : }
832 :
833 : //______________________________________________________________________________
834 : void AliAnalysisAlien::AddRunNumber(const char* run)
835 : {
836 : // Add a run number to the list of runs to be processed.
837 0 : TString runs = run;
838 : TObjString *os;
839 0 : TObjArray *arr = runs.Tokenize(" ");
840 0 : TIter next(arr);
841 0 : TString prefix;
842 0 : prefix.Append(fRunPrefix, fRunPrefix.Index("%d"));
843 0 : while ((os=(TObjString*)next())){
844 0 : if (fRunNumbers.Length()) fRunNumbers += " ";
845 0 : fRunNumbers += Form("%s%s", prefix.Data(), os->GetString().Data());
846 : }
847 0 : delete arr;
848 0 : }
849 :
850 : //______________________________________________________________________________
851 : void AliAnalysisAlien::AddDataFile(const char *lfn)
852 : {
853 : // Adds a data file to the input to be analysed. The file should be a valid LFN
854 : // or point to an existing file in the alien workdir.
855 0 : if (!fInputFiles) fInputFiles = new TObjArray();
856 0 : fInputFiles->Add(new TObjString(lfn));
857 0 : }
858 :
859 : //______________________________________________________________________________
860 : void AliAnalysisAlien::AddExternalPackage(const char *package)
861 : {
862 : // Adds external packages w.r.t to the default ones (root,aliroot and gapi)
863 0 : if (fExternalPackages) fExternalPackages += " ";
864 0 : fExternalPackages += package;
865 0 : }
866 :
867 : //______________________________________________________________________________
868 : Bool_t AliAnalysisAlien::Connect()
869 : {
870 : // Try to connect to AliEn. User needs a valid token and /tmp/gclient_env_$UID sourced.
871 0 : if (gGrid && gGrid->IsConnected()) return kTRUE;
872 0 : if (fProductionMode) return kTRUE;
873 0 : if (!gGrid) {
874 0 : Info("Connect", "Trying to connect to AliEn ...");
875 0 : TGrid::Connect("alien://");
876 0 : }
877 0 : if (!gGrid || !gGrid->IsConnected()) {
878 0 : Error("Connect", "Did not managed to connect to AliEn. Make sure you have a valid token.");
879 0 : return kFALSE;
880 : }
881 0 : fUser = gGrid->GetUser();
882 0 : Info("Connect", "\n##### Connected to AliEn as user %s. Setting analysis user to <%s>", fUser.Data(), fUser.Data());
883 0 : return kTRUE;
884 0 : }
885 :
886 : //______________________________________________________________________________
887 : void AliAnalysisAlien::CdWork()
888 : {
889 : // Check validity of alien workspace. Create directory if possible.
890 0 : if (!Connect()) {
891 0 : Error("CdWork", "Alien connection required");
892 0 : return;
893 : }
894 0 : TString homedir = gGrid->GetHomeDirectory();
895 0 : TString workdir = homedir + fGridWorkingDir;
896 0 : if (DirectoryExists(workdir)) {
897 0 : gGrid->Cd(workdir);
898 0 : return;
899 : }
900 : // Work directory not existing - create it
901 0 : gGrid->Cd(homedir);
902 0 : if (gGrid->Mkdir(workdir, "-p")) {
903 0 : gGrid->Cd(fGridWorkingDir);
904 0 : Info("CdWork", "\n##### Created alien working directory %s", fGridWorkingDir.Data());
905 : } else {
906 0 : Warning("CdWork", "Working directory %s cannot be created.\n Using %s instead.",
907 0 : workdir.Data(), homedir.Data());
908 0 : fGridWorkingDir = "";
909 : }
910 0 : }
911 :
912 : //______________________________________________________________________________
913 : Bool_t AliAnalysisAlien::CheckFileCopy(const char *alienpath)
914 : {
915 : // Check if file copying is possible.
916 0 : if (fProductionMode) return kTRUE;
917 0 : TString salienpath(alienpath);
918 0 : if (salienpath.Contains(" ")) {
919 0 : Error("CheckFileCopy", "path: <%s> contains blancs - FIX IT !",alienpath);
920 0 : return kFALSE;
921 : }
922 0 : if (!Connect()) {
923 0 : Error("CheckFileCopy", "Not connected to AliEn. File copying cannot be tested.");
924 0 : return kFALSE;
925 : }
926 0 : Info("CheckFileCopy", "Checking possibility to copy files to your AliEn home directory... \
927 : \n +++ NOTE: You can disable this via: plugin->SetCheckCopy(kFALSE);");
928 : // Check if alien_CLOSE_SE is defined
929 0 : TString closeSE = gSystem->Getenv("alien_CLOSE_SE");
930 0 : if (!closeSE.IsNull()) {
931 0 : Info("CheckFileCopy", "Your current close storage is pointing to: \
932 0 : \n alien_CLOSE_SE = \"%s\"", closeSE.Data());
933 : } else {
934 0 : Warning("CheckFileCopy", "Your current close storage is empty ! Depending on your location, file copying may fail.");
935 : }
936 : // Check if grid directory exists.
937 0 : if (!DirectoryExists(alienpath)) {
938 0 : Error("CheckFileCopy", "Alien path %s does not seem to exist", alienpath);
939 0 : return kFALSE;
940 : }
941 0 : TString stest = "plugin_test_copy";
942 0 : TFile f(stest, "RECREATE");
943 : // User may not have write permissions to current directory
944 0 : if (f.IsZombie()) {
945 0 : Error("CheckFileCopy", "Cannot create local test file. Do you have write access to current directory: <%s> ?",
946 0 : gSystem->WorkingDirectory());
947 0 : return kFALSE;
948 : }
949 0 : f.Close();
950 0 : if (FileExists(Form("alien://%s/%s",alienpath, stest.Data()))) gGrid->Rm(Form("alien://%s/%s",alienpath, stest.Data()));
951 0 : if (!TFile::Cp(stest.Data(), Form("alien://%s/%s",alienpath, stest.Data()))) {
952 0 : Error("CheckFileCopy", "Cannot copy files to Alien destination: <%s> This may be temporary, or: \
953 : \n# 1. Make sure you have write permissions there. If this is the case: \
954 : \n# 2. Check the storage availability at: http://alimonitor.cern.ch/stats?page=SE/table \
955 : \n# Do: export alien_CLOSE_SE=\"working_disk_SE\" \
956 : \n# To make this permanent put in in your .bashrc (in .alienshrc is not enough) \
957 : \n# Redo token: rm /tmp/x509up_u$UID then: alien-token-init <username>", alienpath);
958 0 : gSystem->Unlink(stest.Data());
959 0 : return kFALSE;
960 : }
961 0 : gSystem->Unlink(stest.Data());
962 0 : gGrid->Rm(Form("%s/%s",alienpath,stest.Data()));
963 0 : Info("CheckFileCopy", "### ...SUCCESS ###");
964 0 : return kTRUE;
965 0 : }
966 :
967 : //______________________________________________________________________________
968 : Bool_t AliAnalysisAlien::CheckInputData()
969 : {
970 : // Check validity of input data. If necessary, create xml files.
971 0 : if (fProductionMode) return kTRUE;
972 0 : if (!fInputFiles && !fRunNumbers.Length() && !fRunRange[0]) {
973 0 : if (!fGridDataDir.Length()) {
974 0 : Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
975 0 : return kFALSE;
976 : }
977 0 : if (fMergeViaJDL) {
978 0 : Error("CheckInputData", "Merging via jdl works only with run numbers, run range or provided xml");
979 0 : return kFALSE;
980 : }
981 0 : Info("CheckInputData", "Analysis will make a single xml for base data directory %s",fGridDataDir.Data());
982 0 : if (fDataPattern.Contains("tag") && TestBit(AliAnalysisGrid::kTest))
983 0 : TObject::SetBit(AliAnalysisGrid::kUseTags, kTRUE); // ADDED (fix problem in determining the tag usage in test mode)
984 0 : return kTRUE;
985 : }
986 : // Process declared files
987 : Bool_t isCollection = kFALSE;
988 : Bool_t isXml = kFALSE;
989 : Bool_t useTags = kFALSE;
990 : Bool_t checked = kFALSE;
991 0 : if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
992 0 : TString file;
993 0 : TString workdir = gGrid->GetHomeDirectory();
994 0 : workdir += fGridWorkingDir;
995 0 : if (fInputFiles) {
996 : TObjString *objstr;
997 0 : TIter next(fInputFiles);
998 0 : while ((objstr=(TObjString*)next())) {
999 0 : file = workdir;
1000 0 : file += "/";
1001 0 : file += objstr->GetString();
1002 : // Store full lfn path
1003 0 : if (FileExists(file)) objstr->SetString(file);
1004 : else {
1005 0 : file = objstr->GetName();
1006 0 : if (!FileExists(objstr->GetName())) {
1007 0 : Error("CheckInputData", "Data file %s not found or not in your working dir: %s",
1008 0 : objstr->GetName(), workdir.Data());
1009 0 : return kFALSE;
1010 : }
1011 : }
1012 0 : Bool_t iscoll, isxml, usetags;
1013 0 : CheckDataType(file, iscoll, isxml, usetags);
1014 0 : if (!checked) {
1015 : checked = kTRUE;
1016 0 : isCollection = iscoll;
1017 0 : isXml = isxml;
1018 0 : useTags = usetags;
1019 0 : TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
1020 : } else {
1021 0 : if ((iscoll != isCollection) || (isxml != isXml) || (usetags != useTags)) {
1022 0 : Error("CheckInputData", "Some conflict was found in the types of inputs");
1023 0 : return kFALSE;
1024 : }
1025 : }
1026 0 : }
1027 0 : }
1028 : // Process requested run numbers
1029 0 : if (!fRunNumbers.Length() && !fRunRange[0]) return kTRUE;
1030 : // Check validity of alien data directory
1031 0 : if (!fGridDataDir.Length()) {
1032 0 : Error("CkeckInputData", "AliEn path to base data directory must be set.\n = Use: SetGridDataDir()");
1033 0 : return kFALSE;
1034 : }
1035 0 : if (!DirectoryExists(fGridDataDir)) {
1036 0 : Error("CheckInputData", "Data directory %s not existing.", fGridDataDir.Data());
1037 0 : return kFALSE;
1038 : }
1039 0 : if (isCollection) {
1040 0 : Error("CheckInputData", "You are using raw AliEn collections as input. Cannot process run numbers.");
1041 0 : return kFALSE;
1042 : }
1043 :
1044 0 : if (checked && !isXml) {
1045 0 : Error("CheckInputData", "Cannot mix processing of full runs with non-xml files");
1046 0 : return kFALSE;
1047 : }
1048 : // Check validity of run number(s)
1049 : TObjArray *arr;
1050 : TObjString *os;
1051 0 : TString format;
1052 : Int_t nruns = 0;
1053 0 : TString schunk, schunk2;
1054 0 : TString path;
1055 0 : if (!checked) {
1056 : checked = kTRUE;
1057 0 : useTags = fDataPattern.Contains("tag");
1058 0 : TObject::SetBit(AliAnalysisGrid::kUseTags, useTags);
1059 : }
1060 0 : if (useTags != fDataPattern.Contains("tag")) {
1061 0 : Error("CheckInputData", "Cannot mix input files using/not using tags");
1062 0 : return kFALSE;
1063 : }
1064 0 : if (fRunNumbers.Length()) {
1065 0 : Info("CheckDataType", "Using supplied run numbers (run ranges are ignored)");
1066 0 : arr = fRunNumbers.Tokenize(" ");
1067 0 : TIter next(arr);
1068 0 : while ((os=(TObjString*)next())) {
1069 0 : path = Form("%s/%s ", fGridDataDir.Data(), os->GetString().Data());
1070 0 : if (!DirectoryExists(path)) {
1071 0 : Warning("CheckInputData", "Run number %s not found in path: <%s>", os->GetString().Data(), path.Data());
1072 0 : continue;
1073 : }
1074 0 : path = Form("%s/%s.xml", workdir.Data(),os->GetString().Data());
1075 0 : TString msg = "\n##### file: ";
1076 0 : msg += path;
1077 0 : msg += " type: xml_collection;";
1078 0 : if (useTags) msg += " using_tags: Yes";
1079 0 : else msg += " using_tags: No";
1080 0 : Info("CheckDataType", "%s", msg.Data());
1081 0 : if (fNrunsPerMaster<2) {
1082 0 : AddDataFile(Form("%s.xml", os->GetString().Data()));
1083 0 : } else {
1084 0 : nruns++;
1085 0 : if (((nruns-1)%fNrunsPerMaster) == 0) {
1086 0 : schunk = os->GetString();
1087 0 : }
1088 0 : if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) continue;
1089 0 : schunk += Form("_%s.xml", os->GetString().Data());
1090 0 : AddDataFile(schunk);
1091 : }
1092 0 : }
1093 0 : delete arr;
1094 0 : } else {
1095 0 : Info("CheckDataType", "Using run range [%d, %d]", fRunRange[0], fRunRange[1]);
1096 0 : for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
1097 0 : format = Form("%%s/%s ", fRunPrefix.Data());
1098 0 : path = Form(format.Data(), fGridDataDir.Data(), irun);
1099 0 : if (!DirectoryExists(path)) {
1100 : continue;
1101 : }
1102 0 : format = Form("%%s/%s.xml", fRunPrefix.Data());
1103 0 : path = Form(format.Data(), workdir.Data(),irun);
1104 0 : TString msg = "\n##### file: ";
1105 0 : msg += path;
1106 0 : msg += " type: xml_collection;";
1107 0 : if (useTags) msg += " using_tags: Yes";
1108 0 : else msg += " using_tags: No";
1109 0 : Info("CheckDataType", "%s", msg.Data());
1110 0 : if (fNrunsPerMaster<2) {
1111 0 : format = Form("%s.xml", fRunPrefix.Data());
1112 0 : AddDataFile(Form(format.Data(),irun));
1113 : } else {
1114 0 : nruns++;
1115 0 : if (((nruns-1)%fNrunsPerMaster) == 0) {
1116 0 : schunk = Form(fRunPrefix.Data(),irun);
1117 : }
1118 0 : format = Form("_%s.xml", fRunPrefix.Data());
1119 0 : schunk2 = Form(format.Data(), irun);
1120 0 : if ((nruns%fNrunsPerMaster)!=0 && irun != fRunRange[1]) continue;
1121 0 : schunk += schunk2;
1122 0 : AddDataFile(schunk);
1123 : }
1124 0 : }
1125 0 : if (!fInputFiles) {
1126 0 : schunk += schunk2;
1127 0 : AddDataFile(schunk);
1128 : }
1129 : }
1130 0 : return kTRUE;
1131 0 : }
1132 :
1133 : //______________________________________________________________________________
1134 : Int_t AliAnalysisAlien::CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output, const char *archivefile, const char *outputdir)
1135 : {
1136 : // Copy data from the given grid directory according a pattern and make a local
1137 : // dataset.
1138 : // archivefile (optional) results in that the archive containing the file <pattern> is copied. archivefile can contain a list of files (semicolon-separated) which are all copied
1139 0 : if (!Connect()) {
1140 0 : Error("CopyLocalDataset", "Cannot copy local dataset with no grid connection");
1141 0 : return 0;
1142 : }
1143 0 : if (!DirectoryExists(griddir)) {
1144 0 : Error("CopyLocalDataset", "Data directory %s not existing.", griddir);
1145 0 : return 0;
1146 : }
1147 0 : TString command = Form("find -z -l %d %s %s", nfiles, griddir, pattern);
1148 0 : printf("Running command: %s\n", command.Data());
1149 0 : TGridResult *res = gGrid->Command(command);
1150 0 : Int_t nfound = res->GetEntries();
1151 0 : if (!nfound) {
1152 0 : Error("CopyLocalDataset", "No file found in <%s> having pattern <%s>", griddir, pattern);
1153 0 : return 0;
1154 : }
1155 0 : printf("... found %d files. Copying locally ...\n", nfound);
1156 :
1157 : // archives
1158 : TObjArray* additionalArchives = 0;
1159 0 : if (strlen(archivefile) > 0 && TString(archivefile).Contains(";")) {
1160 0 : additionalArchives = TString(archivefile).Tokenize(";");
1161 0 : archivefile = additionalArchives->At(0)->GetName();
1162 0 : additionalArchives->RemoveAt(0);
1163 0 : additionalArchives->Compress();
1164 : }
1165 :
1166 : // Copy files locally
1167 0 : ofstream out;
1168 0 : out.open(output, ios::out);
1169 : TMap *map;
1170 0 : TString turl, dirname, filename, temp;
1171 0 : TString cdir = gSystem->WorkingDirectory();
1172 0 : gSystem->MakeDirectory(outputdir);
1173 0 : gSystem->ChangeDirectory(outputdir);
1174 : Int_t ncopied = 0;
1175 0 : for (Int_t i=0; i<nfound; i++) {
1176 0 : map = (TMap*)res->At(i);
1177 0 : turl = map->GetValue("turl")->GetName();
1178 0 : filename = gSystem->BaseName(turl.Data());
1179 0 : dirname = gSystem->DirName(turl.Data());
1180 0 : dirname = gSystem->BaseName(dirname.Data());
1181 0 : gSystem->MakeDirectory(dirname);
1182 :
1183 0 : TString source(turl);
1184 0 : TString targetFileName(filename);
1185 :
1186 0 : if (strlen(archivefile) > 0) {
1187 : // TODO here the archive in which the file resides should be determined
1188 : // however whereis returns only a guid, and guid2lfn does not work
1189 : // Therefore we use the one provided as argument for now
1190 0 : source = Form("%s/%s", gSystem->DirName(source.Data()), archivefile);
1191 0 : targetFileName = archivefile;
1192 : }
1193 0 : if (TFile::Cp(source, Form("file:./%s/%s", dirname.Data(), targetFileName.Data()))) {
1194 : Bool_t success = kTRUE;
1195 0 : if (additionalArchives) {
1196 0 : for (Int_t j=0; j<additionalArchives->GetEntriesFast(); j++) {
1197 0 : TString target;
1198 0 : target.Form("./%s/%s", dirname.Data(), additionalArchives->At(j)->GetName());
1199 0 : gSystem->MakeDirectory(gSystem->DirName(target));
1200 0 : success &= TFile::Cp(Form("%s/%s", gSystem->DirName(source.Data()), additionalArchives->At(j)->GetName()), Form("file:%s", target.Data()));
1201 0 : }
1202 0 : }
1203 :
1204 0 : if (success) {
1205 0 : if (strlen(archivefile) > 0) targetFileName = Form("%s#%s", targetFileName.Data(), gSystem->BaseName(turl.Data()));
1206 0 : out << cdir << Form("/%s/%s/%s", outputdir, dirname.Data(), targetFileName.Data()) << endl;
1207 0 : ncopied++;
1208 0 : }
1209 0 : }
1210 0 : }
1211 0 : gSystem->ChangeDirectory(cdir);
1212 0 : delete res;
1213 0 : delete additionalArchives;
1214 : return ncopied;
1215 0 : }
1216 :
1217 : //______________________________________________________________________________
1218 : Bool_t AliAnalysisAlien::CreateDataset(const char *pattern)
1219 : {
1220 : // Create dataset for the grid data directory + run number.
1221 : const Int_t gMaxEntries = 15000;
1222 0 : if (fProductionMode || TestBit(AliAnalysisGrid::kOffline)) return kTRUE;
1223 0 : if (!Connect()) {
1224 0 : Error("CreateDataset", "Cannot create dataset with no grid connection");
1225 0 : return kFALSE;
1226 : }
1227 :
1228 : // Cd workspace
1229 0 : if (!TestBit(AliAnalysisGrid::kTest)) CdWork();
1230 0 : TString workdir = gGrid->GetHomeDirectory();
1231 0 : workdir += fGridWorkingDir;
1232 :
1233 : // Compose the 'find' command arguments
1234 0 : TString format;
1235 0 : TString command;
1236 0 : TString delimiter = pattern;
1237 0 : delimiter.Strip();
1238 0 : if (delimiter.Contains(" ")) delimiter = "";
1239 0 : else delimiter = " ";
1240 0 : TString options = "-x collection ";
1241 0 : if (TestBit(AliAnalysisGrid::kTest)) options += Form("-l %d ", fNtestFiles);
1242 0 : else options += Form("-l %d ", gMaxEntries); // Protection for the find command
1243 0 : TString conditions = "";
1244 : Int_t nstart = 0;
1245 : Int_t ncount = 0;
1246 : Int_t stage = 0;
1247 0 : TString file;
1248 0 : TString path;
1249 : Int_t nruns = 0;
1250 0 : TString schunk, schunk2;
1251 : TGridCollection *cbase=0, *cadd=0;
1252 0 : if (!fRunNumbers.Length() && !fRunRange[0]) {
1253 0 : if (fInputFiles && fInputFiles->GetEntries()) return kTRUE;
1254 : // Make a single data collection from data directory.
1255 0 : path = fGridDataDir;
1256 0 : if (!DirectoryExists(path)) {
1257 0 : Error("CreateDataset", "Path to data directory %s not valid",fGridDataDir.Data());
1258 0 : return kFALSE;
1259 : }
1260 : // CdWork();
1261 0 : if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1262 0 : else file = Form("%s.xml", gSystem->BaseName(path));
1263 : // cholm - Identical loop - should be put in common function for code simplification
1264 0 : while (1) {
1265 : ncount = 0;
1266 0 : stage++;
1267 0 : if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1268 0 : command = "alien_find ";
1269 0 : command += Form("%s -o %d ",options.Data(), nstart);
1270 0 : command += path;
1271 0 : command += delimiter;
1272 0 : command += pattern;
1273 0 : command += conditions;
1274 0 : printf("command: %s\n", command.Data());
1275 : //TGridResult *res = gGrid->Command(command);
1276 : //if (res) delete res;
1277 : // Write standard output to file
1278 : // Write standard error to null device
1279 0 : gSystem->Exec(Form("%s > __tmp%d__%s 2>/dev/null", command.Data(), stage, file.Data()));
1280 : //gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage, file.Data()));
1281 0 : Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1282 : Bool_t nullFile = kFALSE;
1283 0 : if (!hasGrep) {
1284 0 : Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1285 : } else {
1286 0 : nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1287 0 : if (nullFile) {
1288 0 : Error("CreateDataset","Dataset %s produced by the previous find command is empty !", file.Data());
1289 0 : gSystem->Exec("rm -f __tmp*");
1290 0 : return kFALSE;
1291 : }
1292 0 : TString line;
1293 0 : ifstream in;
1294 0 : in.open("__tmp__");
1295 0 : in >> line;
1296 0 : in.close();
1297 0 : gSystem->Exec("rm -f __tmp__");
1298 0 : ncount = line.Atoi();
1299 0 : }
1300 0 : }
1301 0 : if (ncount == gMaxEntries) {
1302 0 : Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1303 0 : cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1304 0 : if (!cbase) cbase = cadd;
1305 : else {
1306 : // cholm - Avoid using very slow TAlienCollection
1307 : // cbase->Add(cadd);
1308 : // cholm - Use AddFast (via interpreter)
1309 0 : gROOT->ProcessLine(Form("((TAlienCollection*)%p)->AddFast((TGridCollection*)%p)",cbase,cadd));
1310 0 : delete cadd;
1311 : }
1312 0 : nstart += ncount;
1313 : } else {
1314 0 : if (cbase) {
1315 0 : cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1316 0 : printf("... please wait - TAlienCollection::Add() scales badly...\n");
1317 : // cholm - Avoid using very slow TAlienCollection
1318 : // cbase->Add(cadd);
1319 : // cholm - Use AddFast (via interpreter)
1320 0 : gROOT->ProcessLine(Form("((TAlienCollection*)%p)->AddFast((TGridCollection*)%p)",cbase,cadd));
1321 0 : delete cadd;
1322 0 : cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1323 0 : delete cbase; cbase = 0;
1324 0 : } else {
1325 0 : TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1326 : }
1327 0 : gSystem->Exec("rm -f __tmp*");
1328 0 : Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1329 : break;
1330 : }
1331 : }
1332 0 : Bool_t fileExists = FileExists(file);
1333 0 : if (!TestBit(AliAnalysisGrid::kTest) && (!fileExists || fOverwriteMode)) {
1334 : // Copy xml file to alien space
1335 0 : if (fileExists) gGrid->Rm(file);
1336 0 : TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1337 0 : if (!FileExists(file)) {
1338 0 : Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1339 0 : return kFALSE;
1340 : }
1341 : // Update list of files to be processed.
1342 : }
1343 0 : AddDataFile(Form("%s/%s", workdir.Data(), file.Data()));
1344 0 : return kTRUE;
1345 : }
1346 : // Several runs
1347 : Bool_t nullResult = kTRUE;
1348 0 : if (fRunNumbers.Length()) {
1349 0 : TObjArray *arr = fRunNumbers.Tokenize(" ");
1350 : TObjString *os;
1351 0 : TIter next(arr);
1352 0 : while ((os=(TObjString*)next())) {
1353 : nstart = 0;
1354 : stage = 0;
1355 0 : path = Form("%s/%s", fGridDataDir.Data(), os->GetString().Data());
1356 0 : if (!DirectoryExists(path)) continue;
1357 : // CdWork();
1358 0 : if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1359 0 : else file = Form("%s.xml", os->GetString().Data());
1360 : // cholm - Identical loop - should be put in common function for code simplification
1361 : // If local collection file does not exist, create it via 'find' command.
1362 0 : while (1) {
1363 : ncount = 0;
1364 0 : stage++;
1365 0 : if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1366 0 : command = "alien_find ";
1367 0 : command += Form("%s -o %d ",options.Data(), nstart);
1368 0 : command += path;
1369 0 : command += delimiter;
1370 0 : command += pattern;
1371 0 : command += conditions;
1372 : //TGridResult *res = gGrid->Command(command);
1373 : //if (res) delete res;
1374 : // Write standard output to file
1375 : //gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1376 0 : gSystem->Exec(Form("%s > __tmp%d__%s 2>/dev/null", command.Data(), stage, file.Data()));
1377 0 : Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1378 : Bool_t nullFile = kFALSE;
1379 0 : if (!hasGrep) {
1380 0 : Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1381 : } else {
1382 0 : nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1383 0 : if (nullFile) {
1384 0 : Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1385 0 : gSystem->Exec("rm -f __tmp*");
1386 0 : fRunNumbers.ReplaceAll(os->GetString().Data(), "");
1387 0 : break;
1388 : }
1389 0 : TString line;
1390 0 : ifstream in;
1391 0 : in.open("__tmp__");
1392 0 : in >> line;
1393 0 : in.close();
1394 0 : gSystem->Exec("rm -f __tmp__");
1395 0 : ncount = line.Atoi();
1396 0 : }
1397 : nullResult = kFALSE;
1398 0 : }
1399 0 : if (ncount == gMaxEntries) {
1400 0 : Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1401 0 : if (fNrunsPerMaster > 1) {
1402 0 : Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1403 0 : file.Data(),gMaxEntries);
1404 0 : return kFALSE;
1405 : }
1406 0 : cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1407 0 : if (!cbase) cbase = cadd;
1408 : else {
1409 : // cholm - Avoid using very slow TAlienCollection
1410 : // cbase->Add(cadd);
1411 : // cholm - Use AddFast (via interpreter)
1412 0 : gROOT->ProcessLine(Form("((TAlienCollection*)%p)->AddFast((TGridCollection*)%p)",cbase,cadd));
1413 :
1414 0 : delete cadd;
1415 : }
1416 0 : nstart += ncount;
1417 : } else {
1418 0 : if (cbase && fNrunsPerMaster<2) {
1419 0 : cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1420 0 : printf("... please wait - TAlienCollection::Add() scales badly...\n");
1421 : // cholm - Avoid using very slow TAlienCollection
1422 : // cbase->Add(cadd);
1423 : // cholm - Use AddFast (via interpreter)
1424 0 : gROOT->ProcessLine(Form("((TAlienCollection*)%p)->AddFast((TGridCollection*)%p)",cbase,cadd));
1425 0 : delete cadd;
1426 0 : cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1427 0 : delete cbase; cbase = 0;
1428 0 : } else {
1429 0 : TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1430 : }
1431 0 : gSystem->Exec("rm -f __tmp*");
1432 0 : Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1433 : break;
1434 : }
1435 : }
1436 0 : if (TestBit(AliAnalysisGrid::kTest)) break;
1437 : // Check if there is one run per master job.
1438 0 : if (fNrunsPerMaster<2) {
1439 0 : if (FileExists(file)) {
1440 0 : if (fOverwriteMode) gGrid->Rm(file);
1441 : else {
1442 0 : Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1443 : continue;
1444 : }
1445 : }
1446 : // Copy xml file to alien space
1447 0 : TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1448 0 : if (!FileExists(file)) {
1449 0 : Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1450 0 : delete arr;
1451 0 : return kFALSE;
1452 : }
1453 : } else {
1454 0 : nruns++;
1455 0 : if (((nruns-1)%fNrunsPerMaster) == 0) {
1456 0 : schunk = os->GetString();
1457 0 : cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1458 0 : } else {
1459 0 : cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1460 0 : printf(" Merging collection <%s> into masterjob input...\n", file.Data());
1461 : // cholm - Avoid using very slow TAlienCollection
1462 : // cbase->Add(cadd);
1463 : // cholm - Use AddFast (via interpreter)
1464 0 : gROOT->ProcessLine(Form("((TAlienCollection*)%p)->AddFast((TGridCollection*)%p)",cbase,cadd));
1465 0 : delete cadd;
1466 : }
1467 0 : if ((nruns%fNrunsPerMaster)!=0 && os!=arr->Last()) {
1468 : continue;
1469 : }
1470 0 : schunk += Form("_%s.xml", os->GetString().Data());
1471 0 : if (FileExists(schunk)) {
1472 0 : if (fOverwriteMode) gGrid->Rm(file);
1473 : else {
1474 0 : Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1475 : continue;
1476 : }
1477 : }
1478 0 : printf("Exporting merged collection <%s> and copying to AliEn\n", schunk.Data());
1479 0 : cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
1480 0 : TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1481 0 : if (!FileExists(schunk)) {
1482 0 : Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1483 0 : delete arr;
1484 0 : return kFALSE;
1485 : }
1486 : }
1487 : }
1488 0 : delete arr;
1489 0 : if (nullResult) {
1490 0 : Error("CreateDataset", "No valid dataset corresponding to the query!");
1491 0 : return kFALSE;
1492 : }
1493 0 : } else {
1494 : // Process a full run range.
1495 0 : for (Int_t irun=fRunRange[0]; irun<=fRunRange[1]; irun++) {
1496 0 : format = Form("%%s/%s", fRunPrefix.Data());
1497 : nstart = 0;
1498 : stage = 0;
1499 0 : path = Form(format.Data(), fGridDataDir.Data(), irun);
1500 0 : if (!DirectoryExists(path)) continue;
1501 : // CdWork();
1502 0 : format = Form("%s.xml", fRunPrefix.Data());
1503 0 : if (TestBit(AliAnalysisGrid::kTest)) file = "wn.xml";
1504 0 : else file = Form(format.Data(), irun);
1505 0 : if (FileExists(file) && fNrunsPerMaster<2 && !TestBit(AliAnalysisGrid::kTest)) {
1506 0 : if (fOverwriteMode) gGrid->Rm(file);
1507 : else {
1508 0 : Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1509 : continue;
1510 : }
1511 : }
1512 : // cholm - Identical loop - should be put in common function for code simplification
1513 : // If local collection file does not exist, create it via 'find' command.
1514 0 : while (1) {
1515 : ncount = 0;
1516 0 : stage++;
1517 0 : if (gSystem->AccessPathName(file) || TestBit(AliAnalysisGrid::kTest) || fOverwriteMode) {
1518 0 : command = "alien_find ";
1519 0 : command += Form("%s -o %d ",options.Data(), nstart);
1520 0 : command += path;
1521 0 : command += delimiter;
1522 0 : command += pattern;
1523 0 : command += conditions;
1524 : //TGridResult *res = gGrid->Command(command);
1525 : //if (res) delete res;
1526 : // Write standard output to file
1527 0 : gSystem->Exec(Form("%s > __tmp%d__%s 2>/dev/null", command.Data(), stage, file.Data()));
1528 : //gROOT->ProcessLine(Form("gGrid->Stdout(); > __tmp%d__%s", stage,file.Data()));
1529 0 : Bool_t hasGrep = (gSystem->Exec("grep --version 2>/dev/null > /dev/null")==0)?kTRUE:kFALSE;
1530 : Bool_t nullFile = kFALSE;
1531 0 : if (!hasGrep) {
1532 0 : Warning("CreateDataset", "'grep' command not available on this system - cannot validate the result of the grid 'find' command");
1533 : } else {
1534 0 : nullFile = (gSystem->Exec(Form("grep -c /event __tmp%d__%s 2>/dev/null > __tmp__",stage,file.Data()))==0)?kFALSE:kTRUE;
1535 0 : if (nullFile) {
1536 0 : Warning("CreateDataset","Dataset %s produced by: <%s> is empty !", file.Data(), command.Data());
1537 0 : gSystem->Exec("rm -f __tmp*");
1538 0 : break;
1539 : }
1540 0 : TString line;
1541 0 : ifstream in;
1542 0 : in.open("__tmp__");
1543 0 : in >> line;
1544 0 : in.close();
1545 0 : gSystem->Exec("rm -f __tmp__");
1546 0 : ncount = line.Atoi();
1547 0 : }
1548 : nullResult = kFALSE;
1549 0 : }
1550 0 : if (ncount == gMaxEntries) {
1551 0 : Info("CreateDataset", "Dataset %s has more than 15K entries. Trying to merge...", file.Data());
1552 0 : if (fNrunsPerMaster > 1) {
1553 0 : Error("CreateDataset", "File %s has more than %d entries. Please set the number of runs per master to 1 !",
1554 0 : file.Data(),gMaxEntries);
1555 0 : return kFALSE;
1556 : }
1557 0 : cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1558 0 : if (!cbase) cbase = cadd;
1559 : else {
1560 : // cholm - Avoid using very slow TAlienCollection
1561 : // cbase->Add(cadd);
1562 : // cholm - Use AddFast (via interpreter)
1563 0 : gROOT->ProcessLine(Form("((TAlienCollection*)%p)->AddFast((TGridCollection*)%p)",cbase,cadd));
1564 0 : delete cadd;
1565 : }
1566 0 : nstart += ncount;
1567 : } else {
1568 0 : if (cbase && fNrunsPerMaster<2) {
1569 0 : cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"__tmp%d__%s\", 1000000);",stage,file.Data()));
1570 0 : printf("... please wait - TAlienCollection::Add() scales badly...\n");
1571 : // cholm - Avoid using very slow TAlienCollection
1572 : // cbase->Add(cadd);
1573 : // cholm - Use AddFast (via interpreter)
1574 0 : gROOT->ProcessLine(Form("((TAlienCollection*)%p)->AddFast((TGridCollection*)%p)",cbase,cadd));
1575 0 : delete cadd;
1576 0 : cbase->ExportXML(Form("file://%s", file.Data()),kFALSE,kFALSE, file, "Merged entries for a run");
1577 0 : delete cbase; cbase = 0;
1578 0 : } else {
1579 0 : TFile::Cp(Form("__tmp%d__%s",stage, file.Data()), file.Data());
1580 : }
1581 0 : Info("CreateDataset", "Created dataset %s with %d files", file.Data(), nstart+ncount);
1582 : break;
1583 : }
1584 : }
1585 0 : if (TestBit(AliAnalysisGrid::kTest)) break;
1586 : // Check if there is one run per master job.
1587 0 : if (fNrunsPerMaster<2) {
1588 0 : if (FileExists(file)) {
1589 0 : if (fOverwriteMode) gGrid->Rm(file);
1590 : else {
1591 0 : Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", file.Data());
1592 : continue;
1593 : }
1594 : }
1595 : // Copy xml file to alien space
1596 0 : TFile::Cp(Form("file:%s",file.Data()), Form("alien://%s/%s",workdir.Data(), file.Data()));
1597 0 : if (!FileExists(file)) {
1598 0 : Error("CreateDataset", "Command %s did NOT succeed", command.Data());
1599 0 : return kFALSE;
1600 : }
1601 : } else {
1602 0 : nruns++;
1603 : // Check if the collection for the chunk exist locally.
1604 0 : Int_t nchunk = (nruns-1)/fNrunsPerMaster;
1605 0 : if (FileExists(fInputFiles->At(nchunk)->GetName())) {
1606 0 : if (fOverwriteMode) gGrid->Rm(fInputFiles->At(nchunk)->GetName());
1607 0 : else continue;
1608 : }
1609 0 : printf(" Merging collection <%s> into %d runs chunk...\n",file.Data(),fNrunsPerMaster);
1610 0 : if (((nruns-1)%fNrunsPerMaster) == 0) {
1611 0 : schunk = Form(fRunPrefix.Data(), irun);
1612 0 : cbase = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1613 0 : } else {
1614 0 : cadd = (TGridCollection*)gROOT->ProcessLine(Form("new TAlienCollection(\"%s\", 1000000);",file.Data()));
1615 : // cholm - Avoid using very slow TAlienCollection
1616 : // cbase->Add(cadd);
1617 : // cholm - Use AddFast (via interpreter)
1618 0 : gROOT->ProcessLine(Form("((TAlienCollection*)%p)->AddFast((TGridCollection*)%p)",cbase,cadd));
1619 0 : delete cadd;
1620 : }
1621 0 : format = Form("%%s_%s.xml", fRunPrefix.Data());
1622 0 : schunk2 = Form(format.Data(), schunk.Data(), irun);
1623 0 : if ((nruns%fNrunsPerMaster)!=0 && irun!=fRunRange[1] && schunk2 != fInputFiles->Last()->GetName()) {
1624 0 : continue;
1625 : }
1626 0 : schunk = schunk2;
1627 0 : if (FileExists(schunk)) {
1628 0 : if (fOverwriteMode) gGrid->Rm(schunk);
1629 : else {
1630 0 : Info("CreateDataset", "\n##### Dataset %s exist. Skipping creation...", schunk.Data());
1631 0 : continue;
1632 : }
1633 : }
1634 0 : printf("Exporting merged collection <%s> and copying to AliEn.\n", schunk.Data());
1635 0 : cbase->ExportXML(Form("file://%s", schunk.Data()),kFALSE,kFALSE, schunk, "Merged runs");
1636 0 : if (FileExists(schunk)) {
1637 0 : if (fOverwriteMode) gGrid->Rm(schunk);
1638 : else {
1639 0 : Info("CreateDataset", "\n##### Dataset %s exist. Skipping copy...", schunk.Data());
1640 0 : continue;
1641 : }
1642 : }
1643 0 : TFile::Cp(Form("file:%s",schunk.Data()), Form("alien://%s/%s",workdir.Data(), schunk.Data()));
1644 0 : if (!FileExists(schunk)) {
1645 0 : Error("CreateDataset", "Copy command did NOT succeed for %s", schunk.Data());
1646 0 : return kFALSE;
1647 : }
1648 0 : }
1649 : }
1650 0 : if (nullResult) {
1651 0 : Error("CreateDataset", "No valid dataset corresponding to the query!");
1652 0 : return kFALSE;
1653 : }
1654 : }
1655 0 : return kTRUE;
1656 0 : }
1657 :
1658 : //______________________________________________________________________________
1659 : Bool_t AliAnalysisAlien::CreateJDL()
1660 : {
1661 : // Generate a JDL file according to current settings. The name of the file is
1662 : // specified by fJDLName.
1663 0 : AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
1664 : Bool_t error = kFALSE;
1665 : TObjArray *arr = 0;
1666 : Bool_t copy = kTRUE;
1667 0 : if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
1668 : Bool_t generate = kTRUE;
1669 0 : if (TestBit(AliAnalysisGrid::kTest) || TestBit(AliAnalysisGrid::kSubmit)) generate = kFALSE;
1670 0 : if (!Connect()) {
1671 0 : Error("CreateJDL", "Alien connection required");
1672 0 : return kFALSE;
1673 : }
1674 : // Check validity of alien workspace
1675 0 : TString workdir;
1676 0 : if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
1677 0 : if (!fProductionMode && !TestBit(AliAnalysisGrid::kTest)) CdWork();
1678 0 : workdir += fGridWorkingDir;
1679 0 : if (generate) {
1680 : TObjString *os;
1681 0 : if (!fInputFiles && !fMCLoop) {
1682 0 : Error("CreateJDL()", "Define some input files for your analysis.");
1683 : error = kTRUE;
1684 0 : }
1685 : // Compose list of input files
1686 : // Check if output files were defined
1687 0 : if (!fOutputFiles.Length()) {
1688 0 : Error("CreateJDL", "You must define at least one output file");
1689 : error = kTRUE;
1690 0 : }
1691 : // Check if an output directory was defined and valid
1692 0 : if (!fGridOutputDir.Length()) {
1693 0 : Error("CreateJDL", "You must define AliEn output directory");
1694 : error = kTRUE;
1695 0 : } else {
1696 0 : if (!fProductionMode) {
1697 0 : if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1698 0 : if (!DirectoryExists(fGridOutputDir)) {
1699 0 : if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1700 0 : Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1701 : } else {
1702 0 : Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1703 : // error = kTRUE;
1704 : }
1705 : } else {
1706 0 : Warning("CreateJDL", "#### Output directory %s exists! If this contains old data, jobs will fail with ERROR_SV !!! ###", fGridOutputDir.Data());
1707 : }
1708 0 : gGrid->Cd(workdir);
1709 : }
1710 : }
1711 : // Exit if any error up to now
1712 0 : if (error) return kFALSE;
1713 : // Set JDL fields
1714 0 : if (!fUser.IsNull()) {
1715 0 : fGridJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1716 0 : fMergingJDL->SetValue("User", Form("\"%s\"", fUser.Data()));
1717 : }
1718 0 : TString executable = fExecutable;
1719 0 : if (!executable.BeginsWith("/"))
1720 0 : executable.Prepend(Form("%s/", workdir.Data()));
1721 0 : fGridJDL->SetExecutable(executable, "This is the startup script");
1722 0 : TString mergeExec = executable;
1723 0 : mergeExec.ReplaceAll(".sh", "_merge.sh");
1724 0 : fMergingJDL->SetExecutable(mergeExec, "This is the startup script");
1725 0 : mergeExec.ReplaceAll(".sh", ".C");
1726 0 : fMergingJDL->AddToInputSandbox(Form("LF:%s", mergeExec.Data()), "List of input files to be uploaded to workers");
1727 0 : if (!fArguments.IsNull())
1728 0 : fGridJDL->SetArguments(fArguments, "Arguments for the executable command");
1729 0 : if (IsOneStageMerging()) fMergingJDL->SetArguments(fGridOutputDir);
1730 : else {
1731 0 : if (fProductionMode) fMergingJDL->SetArguments("wn.xml $4"); // xml, stage
1732 0 : else fMergingJDL->SetArguments("wn.xml $2"); // xml, stage
1733 : }
1734 :
1735 0 : fGridJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1736 0 : fGridJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1737 0 : fMergingJDL->SetValue("TTL", Form("\"%d\"",fTTL));
1738 0 : fMergingJDL->SetDescription("TTL", Form("Time after which the job is killed (%d min.)", fTTL/60));
1739 :
1740 0 : if (fMaxInitFailed > 0) {
1741 0 : fGridJDL->SetValue("MaxInitFailed", Form("\"%d\"",fMaxInitFailed));
1742 0 : fGridJDL->SetDescription("MaxInitFailed", "Maximum number of first failing jobs to abort the master job");
1743 : }
1744 0 : if (fSplitMaxInputFileNumber > 0 && !fMCLoop) {
1745 0 : fGridJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"", fSplitMaxInputFileNumber));
1746 0 : fGridJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be processed per subjob");
1747 : }
1748 0 : if (!IsOneStageMerging()) {
1749 0 : fMergingJDL->SetValue("SplitMaxInputFileNumber", Form("\"%d\"",fMaxMergeFiles));
1750 0 : fMergingJDL->SetDescription("SplitMaxInputFileNumber", "Maximum number of input files to be merged in one go");
1751 : }
1752 0 : if (fSplitMode.Length()) {
1753 0 : fGridJDL->SetValue("Split", Form("\"%s\"", fSplitMode.Data()));
1754 0 : fGridJDL->SetDescription("Split", "We split per SE or file");
1755 : }
1756 0 : fMergingJDL->SetValue("Split", "\"se\"");
1757 0 : fMergingJDL->SetDescription("Split", "We split per SE for merging in stages");
1758 0 : if (!fAliROOTVersion.IsNull()) {
1759 0 : fGridJDL->AddToPackages("AliRoot", fAliROOTVersion,"VO_ALICE", "List of requested packages");
1760 0 : fMergingJDL->AddToPackages("AliRoot", fAliROOTVersion, "VO_ALICE", "List of requested packages");
1761 : }
1762 0 : if (!fAliPhysicsVersion.IsNull()) {
1763 0 : fGridJDL->AddToPackages("AliPhysics", fAliPhysicsVersion,"VO_ALICE", "List of requested packages");
1764 0 : fMergingJDL->AddToPackages("AliPhysics", fAliPhysicsVersion, "VO_ALICE", "List of requested packages");
1765 : }
1766 0 : if (!fROOTVersion.IsNull()) {
1767 0 : fGridJDL->AddToPackages("ROOT", fROOTVersion);
1768 0 : fMergingJDL->AddToPackages("ROOT", fROOTVersion);
1769 : }
1770 0 : if (!fAPIVersion.IsNull()) {
1771 0 : fGridJDL->AddToPackages("APISCONFIG", fAPIVersion);
1772 0 : fMergingJDL->AddToPackages("APISCONFIG", fAPIVersion);
1773 : }
1774 0 : if (!fExternalPackages.IsNull()) {
1775 0 : arr = fExternalPackages.Tokenize(" ");
1776 0 : TIter next(arr);
1777 0 : while ((os=(TObjString*)next())) {
1778 0 : TString pkgname = os->GetString();
1779 0 : Int_t index = pkgname.Index("::");
1780 0 : TString pkgversion = pkgname(index+2, pkgname.Length());
1781 0 : pkgname.Remove(index);
1782 0 : fGridJDL->AddToPackages(pkgname, pkgversion);
1783 0 : fMergingJDL->AddToPackages(pkgname, pkgversion);
1784 0 : }
1785 0 : delete arr;
1786 0 : }
1787 0 : if (!fMCLoop) {
1788 0 : fGridJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1789 0 : fGridJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
1790 : }
1791 0 : fMergingJDL->SetInputDataListFormat(fInputFormat, "Format of input data");
1792 0 : fMergingJDL->SetInputDataList("wn.xml", "Collection name to be processed on each worker node");
1793 0 : fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), fAnalysisMacro.Data()), "List of input files to be uploaded to workers");
1794 0 : TString analysisFile = fExecutable;
1795 0 : analysisFile.ReplaceAll(".sh", ".root");
1796 0 : fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
1797 0 : fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(),analysisFile.Data()));
1798 0 : if (fAdditionalLibs.Length()) {
1799 0 : arr = fAdditionalLibs.Tokenize(" ");
1800 0 : TIter next(arr);
1801 0 : while ((os=(TObjString*)next())) {
1802 0 : if (os->GetString().Contains(".so") ||
1803 0 : os->GetString().Contains(".dylib")) continue;
1804 0 : fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
1805 0 : fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), os->GetString().Data()));
1806 : }
1807 0 : delete arr;
1808 0 : }
1809 0 : if (fPackages) {
1810 0 : TIter next(fPackages);
1811 : TObject *obj;
1812 0 : while ((obj=next())) {
1813 0 : fGridJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1814 0 : fMergingJDL->AddToInputSandbox(Form("LF:%s/%s", workdir.Data(), obj->GetName()));
1815 : }
1816 0 : }
1817 : const char *comment = "List of output files and archives";
1818 0 : if (fOutputArchive.Length()) {
1819 0 : TString outputArchive = fOutputArchive;
1820 0 : if (!fRegisterExcludes.IsNull()) {
1821 0 : arr = fRegisterExcludes.Tokenize(" ");
1822 0 : TIter next1(arr);
1823 0 : while ((os=(TObjString*)next1())) {
1824 0 : outputArchive.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1825 0 : outputArchive.ReplaceAll(os->GetString(),"");
1826 : }
1827 0 : delete arr;
1828 0 : }
1829 0 : arr = outputArchive.Tokenize(" ");
1830 0 : TIter next(arr);
1831 : Bool_t first = kTRUE;
1832 0 : while ((os=(TObjString*)next())) {
1833 0 : if (!os->GetString().Contains("@") && fCloseSE.Length())
1834 0 : fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
1835 : else
1836 0 : fGridJDL->AddToSet("Output", os->GetString());
1837 0 : if (first) fGridJDL->AddToSetDescription("Output", comment);
1838 : first = kFALSE;
1839 : }
1840 0 : delete arr;
1841 : // Output archive for the merging jdl
1842 0 : if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
1843 0 : outputArchive = "log_archive.zip:std*@disk=1 ";
1844 : // Add normal output files, extra files + terminate files
1845 0 : TString files;
1846 0 : if (IsMergeAOD()) files = GetListOfFiles("outaodextter");
1847 0 : else files = GetListOfFiles("outextter");
1848 : // Do not register files in fRegisterExcludes
1849 0 : if (!fRegisterExcludes.IsNull()) {
1850 0 : arr = fRegisterExcludes.Tokenize(" ");
1851 0 : TIter next1(arr);
1852 0 : while ((os=(TObjString*)next1())) {
1853 0 : files.ReplaceAll(Form("%s,",os->GetString().Data()),"");
1854 0 : files.ReplaceAll(os->GetString(),"");
1855 : }
1856 0 : delete arr;
1857 0 : }
1858 0 : files.ReplaceAll(".root", "*.root");
1859 :
1860 0 : if (mgr->IsCollectThroughput())
1861 0 : outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",files.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
1862 : else
1863 0 : outputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",files.Data(),fNreplicas);
1864 0 : } else {
1865 0 : TString files = fOutputArchive;
1866 0 : files.ReplaceAll(".root", "*.root"); // nreplicas etc should be already atttached by use
1867 0 : outputArchive = files;
1868 0 : }
1869 0 : arr = outputArchive.Tokenize(" ");
1870 0 : TIter next2(arr);
1871 : first = kTRUE;
1872 0 : while ((os=(TObjString*)next2())) {
1873 0 : TString currentfile = os->GetString();
1874 0 : if (!currentfile.Contains("@") && fCloseSE.Length())
1875 0 : fMergingJDL->AddToSet("Output", Form("%s@%s",currentfile.Data(), fCloseSE.Data()));
1876 : else
1877 0 : fMergingJDL->AddToSet("Output", currentfile);
1878 0 : if (first) fMergingJDL->AddToSetDescription("Output", comment);
1879 : first = kFALSE;
1880 0 : }
1881 0 : delete arr;
1882 0 : }
1883 0 : arr = fOutputFiles.Tokenize(",");
1884 0 : TIter next(arr);
1885 : Bool_t first = kTRUE;
1886 0 : while ((os=(TObjString*)next())) {
1887 : // Ignore ouputs in jdl that are also in outputarchive
1888 0 : TString sout = os->GetString();
1889 0 : sout.ReplaceAll("*", "");
1890 0 : sout.ReplaceAll(".root", "");
1891 0 : if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
1892 0 : if (fOutputArchive.Contains(sout)) continue;
1893 : // Ignore fRegisterExcludes
1894 0 : if (fRegisterExcludes.Contains(sout)) continue;
1895 0 : if (!first) comment = NULL;
1896 0 : if (!os->GetString().Contains("@") && fCloseSE.Length())
1897 0 : fGridJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
1898 : else
1899 0 : fGridJDL->AddToSet("Output", os->GetString());
1900 0 : if (first) fGridJDL->AddToSetDescription("Output", comment);
1901 0 : if (fMergeExcludes.Contains(sout)) continue;
1902 0 : if (!os->GetString().Contains("@") && fCloseSE.Length())
1903 0 : fMergingJDL->AddToSet("Output", Form("%s@%s",os->GetString().Data(), fCloseSE.Data()));
1904 : else
1905 0 : fMergingJDL->AddToSet("Output", os->GetString());
1906 0 : if (first) fMergingJDL->AddToSetDescription("Output", comment);
1907 : first = kFALSE;
1908 0 : }
1909 0 : delete arr;
1910 0 : fGridJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1911 0 : fMergingJDL->SetPrice((UInt_t)fPrice, "AliEn price for this job");
1912 0 : TString validationScript = fValidationScript;
1913 0 : fGridJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1914 0 : validationScript.ReplaceAll(".sh", "_merge.sh");
1915 0 : fMergingJDL->SetValidationCommand(Form("%s/%s", workdir.Data(),validationScript.Data()), "Validation script to be run for each subjob");
1916 0 : if (fMasterResubmitThreshold) {
1917 0 : fGridJDL->SetValue("MasterResubmitThreshold", Form("\"%d%%\"", fMasterResubmitThreshold));
1918 0 : fGridJDL->SetDescription("MasterResubmitThreshold", "Resubmit failed jobs until DONE rate reaches this percentage");
1919 : }
1920 : // Write a jdl with 2 input parameters: collection name and output dir name.
1921 0 : WriteJDL(copy);
1922 0 : }
1923 : // Copy jdl to grid workspace
1924 0 : if (copy) {
1925 : // Check if an output directory was defined and valid
1926 0 : if (!fGridOutputDir.Length()) {
1927 0 : Error("CreateJDL", "You must define AliEn output directory");
1928 0 : return kFALSE;
1929 : } else {
1930 0 : if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s", workdir.Data(), fGridOutputDir.Data());
1931 0 : if (!fProductionMode && !DirectoryExists(fGridOutputDir)) {
1932 0 : if (gGrid->Mkdir(fGridOutputDir,"-p")) {
1933 0 : Info("CreateJDL", "\n##### Created alien output directory %s", fGridOutputDir.Data());
1934 : } else {
1935 0 : Error("CreateJDL", "Could not create alien output directory %s", fGridOutputDir.Data());
1936 0 : return kFALSE;
1937 : }
1938 : }
1939 0 : gGrid->Cd(workdir);
1940 : }
1941 0 : if (TestBit(AliAnalysisGrid::kSubmit)) {
1942 0 : TString mergeJDLName = fExecutable;
1943 0 : mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
1944 0 : TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
1945 0 : TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
1946 0 : if (fProductionMode) {
1947 0 : locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
1948 0 : locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
1949 : }
1950 0 : if (FileExists(locjdl)) gGrid->Rm(locjdl);
1951 0 : if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
1952 0 : Info("CreateJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
1953 0 : if (!copyLocal2Alien("CreateJDL", fJDLName, locjdl))
1954 0 : Fatal("","Terminating");
1955 : // TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
1956 0 : if (fMergeViaJDL) {
1957 0 : Info("CreateJDL", "\n##### Copying merging JDL file <%s> to your AliEn output directory", mergeJDLName.Data());
1958 : // TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
1959 0 : if (!copyLocal2Alien("CreateJDL", mergeJDLName.Data(), locjdl1))
1960 0 : Fatal("","Terminating");
1961 : }
1962 0 : }
1963 0 : if (fAdditionalLibs.Length()) {
1964 0 : arr = fAdditionalLibs.Tokenize(" ");
1965 : TObjString *os;
1966 0 : TIter next(arr);
1967 0 : while ((os=(TObjString*)next())) {
1968 0 : if (os->GetString().Contains(".so") ||
1969 0 : os->GetString().Contains(".dylib")) continue;
1970 0 : Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", os->GetString().Data());
1971 0 : if (FileExists(os->GetString())) gGrid->Rm(os->GetString());
1972 : // TFile::Cp(Form("file:%s",os->GetString().Data()), Form("alien://%s/%s", workdir.Data(), os->GetString().Data()));
1973 0 : if (!copyLocal2Alien("CreateJDL", os->GetString().Data(),
1974 0 : Form("%s/%s", workdir.Data(), os->GetString().Data())))
1975 0 : Fatal("","Terminating");
1976 : }
1977 0 : delete arr;
1978 0 : }
1979 0 : if (fPackages) {
1980 0 : TIter next(fPackages);
1981 : TObject *obj;
1982 0 : while ((obj=next())) {
1983 0 : if (FileExists(obj->GetName())) gGrid->Rm(obj->GetName());
1984 0 : Info("CreateJDL", "\n##### Copying dependency: <%s> to your alien workspace", obj->GetName());
1985 : // TFile::Cp(Form("file:%s",obj->GetName()), Form("alien://%s/%s", workdir.Data(), obj->GetName()));
1986 0 : if (!copyLocal2Alien("CreateJDL",obj->GetName(),
1987 0 : Form("%s/%s", workdir.Data(), obj->GetName())))
1988 0 : Fatal("","Terminating");
1989 : }
1990 0 : }
1991 : }
1992 0 : return kTRUE;
1993 0 : }
1994 :
1995 : //______________________________________________________________________________
1996 : Bool_t AliAnalysisAlien::WriteJDL(Bool_t copy)
1997 : {
1998 : // Writes one or more JDL's corresponding to findex. If findex is negative,
1999 : // all run numbers are considered in one go (jdl). For non-negative indices
2000 : // they correspond to the indices in the array fInputFiles.
2001 0 : if (!fInputFiles && !fMCLoop) return kFALSE;
2002 : TObject *os;
2003 0 : TString workdir;
2004 0 : if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice")) workdir = gGrid->GetHomeDirectory();
2005 0 : workdir += fGridWorkingDir;
2006 0 : TString stageName = "$2";
2007 0 : if (fProductionMode) stageName = "$4";
2008 0 : if (!fMergeDirName.IsNull()) {
2009 0 : fMergingJDL->AddToInputDataCollection(Form("LF:$1/%s/Stage_%s.xml,nodownload",fMergeDirName.Data(),stageName.Data()), "Collection of files to be merged for current stage");
2010 0 : fMergingJDL->SetOutputDirectory(Form("$1/%s/Stage_%s/#alien_counter_03i#",fMergeDirName.Data(),stageName.Data()), "Output directory");
2011 : } else {
2012 0 : fMergingJDL->AddToInputDataCollection(Form("LF:$1/Stage_%s.xml,nodownload",stageName.Data()), "Collection of files to be merged for current stage");
2013 0 : fMergingJDL->SetOutputDirectory(Form("$1/Stage_%s/#alien_counter_03i#",stageName.Data()), "Output directory");
2014 : }
2015 0 : if (fProductionMode) {
2016 0 : TIter next(fInputFiles);
2017 0 : while ((os=next())) {
2018 0 : fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
2019 : }
2020 0 : if (!fOutputToRunNo)
2021 0 : fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_04i#", fGridOutputDir.Data()));
2022 : else
2023 0 : fGridJDL->SetOutputDirectory(fGridOutputDir);
2024 0 : } else {
2025 0 : if (!fRunNumbers.Length() && !fRunRange[0]) {
2026 : // One jdl with no parameters in case input data is specified by name.
2027 0 : TIter next(fInputFiles);
2028 0 : while ((os=next()))
2029 0 : fGridJDL->AddToInputDataCollection(Form("LF:%s,nodownload", os->GetName()), "Input xml collections");
2030 0 : if (!fOutputSingle.IsNull())
2031 0 : fGridJDL->SetOutputDirectory(Form("#alienfulldir#/../%s",fOutputSingle.Data()), "Output directory");
2032 : else {
2033 0 : fGridJDL->SetOutputDirectory(Form("%s/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
2034 : // fMergingJDL->SetOutputDirectory(fGridOutputDir);
2035 : }
2036 0 : } else {
2037 : // One jdl to be submitted with 2 input parameters: data collection name and output dir prefix
2038 0 : fGridJDL->AddToInputDataCollection(Form("LF:%s/$1,nodownload", workdir.Data()), "Input xml collections");
2039 0 : if (!fOutputSingle.IsNull()) {
2040 0 : if (!fOutputToRunNo) fGridJDL->SetOutputDirectory(Form("#alienfulldir#/%s",fOutputSingle.Data()), "Output directory");
2041 0 : else fGridJDL->SetOutputDirectory(Form("%s/$2",fGridOutputDir.Data()), "Output directory");
2042 : } else {
2043 0 : fGridJDL->SetOutputDirectory(Form("%s/$2/#alien_counter_03i#", fGridOutputDir.Data()), "Output directory");
2044 : }
2045 : }
2046 : }
2047 :
2048 : // Generate the JDL as a string
2049 0 : TString sjdl = fGridJDL->Generate();
2050 0 : TString sjdl1 = fMergingJDL->Generate();
2051 : // Final merge jdl
2052 0 : if (!fMergeDirName.IsNull()) {
2053 0 : fMergingJDL->SetOutputDirectory(Form("$1/%s",fMergeDirName.Data()), "Output directory");
2054 0 : fMergingJDL->AddToInputSandbox(Form("LF:$1/%s/Stage_%s.xml",fMergeDirName.Data(),stageName.Data()));
2055 : } else {
2056 0 : fMergingJDL->SetOutputDirectory("$1", "Output directory");
2057 0 : fMergingJDL->AddToInputSandbox(Form("LF:$1/Stage_%s.xml",stageName.Data()));
2058 : }
2059 0 : TString sjdl2 = fMergingJDL->Generate();
2060 : Int_t index, index1;
2061 0 : sjdl.ReplaceAll("\",\"", "\",\n \"");
2062 0 : sjdl.ReplaceAll("(member", "\n (member");
2063 0 : sjdl.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2064 0 : sjdl.ReplaceAll("{", "{\n ");
2065 0 : sjdl.ReplaceAll("};", "\n};");
2066 0 : sjdl.ReplaceAll("{\n \n", "{\n");
2067 0 : sjdl.ReplaceAll("\n\n", "\n");
2068 0 : sjdl.ReplaceAll("OutputDirectory", "OutputDir");
2069 0 : sjdl1.ReplaceAll("\",\"", "\",\n \"");
2070 0 : sjdl1.ReplaceAll("(member", "\n (member");
2071 0 : sjdl1.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2072 0 : sjdl1.ReplaceAll("{", "{\n ");
2073 0 : sjdl1.ReplaceAll("};", "\n};");
2074 0 : sjdl1.ReplaceAll("{\n \n", "{\n");
2075 0 : sjdl1.ReplaceAll("\n\n", "\n");
2076 0 : sjdl1.ReplaceAll("OutputDirectory", "OutputDir");
2077 0 : sjdl2.ReplaceAll("\",\"", "\",\n \"");
2078 0 : sjdl2.ReplaceAll("(member", "\n (member");
2079 0 : sjdl2.ReplaceAll("\",\"VO_", "\",\n \"VO_");
2080 0 : sjdl2.ReplaceAll("{", "{\n ");
2081 0 : sjdl2.ReplaceAll("};", "\n};");
2082 0 : sjdl2.ReplaceAll("{\n \n", "{\n");
2083 0 : sjdl2.ReplaceAll("\n\n", "\n");
2084 0 : sjdl2.ReplaceAll("OutputDirectory", "OutputDir");
2085 0 : sjdl += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
2086 0 : sjdl.Prepend(Form("Jobtag = {\n \"comment:%s\"\n};\n", fJobTag.Data()));
2087 0 : index = sjdl.Index("JDLVariables");
2088 0 : if (index >= 0) sjdl.Insert(index, "\n# JDL variables\n");
2089 0 : sjdl += "Workdirectorysize = {\"5000MB\"};";
2090 0 : sjdl1 += "Workdirectorysize = {\"5000MB\"};";
2091 0 : sjdl1 += "JDLVariables = \n{\n \"Packages\",\n \"OutputDir\"\n};\n";
2092 0 : index = fJobTag.Index(":");
2093 0 : if (index < 0) index = fJobTag.Length();
2094 0 : TString jobTag = fJobTag;
2095 0 : if (fProductionMode) jobTag.Insert(index,"_Stage$4");
2096 0 : sjdl1.Prepend(Form("Jobtag = {\n \"comment:%s_Merging\"\n};\n", jobTag.Data()));
2097 0 : if (fProductionMode) {
2098 0 : sjdl1.Prepend("# Generated merging jdl (production mode) \
2099 : \n# $1 = full alien path to output directory to be merged \
2100 : \n# $2 = train number \
2101 : \n# $3 = production (like LHC10b) \
2102 : \n# $4 = merging stage \
2103 : \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2104 0 : sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2105 0 : sjdl2.Prepend("# Generated merging jdl \
2106 : \n# $1 = full alien path to output directory to be merged \
2107 : \n# $2 = train number \
2108 : \n# $3 = production (like LHC10b) \
2109 : \n# $4 = merging stage \
2110 : \n# Stage_<n>.xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2111 : } else {
2112 0 : sjdl1.Prepend("# Generated merging jdl \
2113 : \n# $1 = full alien path to output directory to be merged \
2114 : \n# $2 = merging stage \
2115 : \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2116 0 : sjdl2.Prepend(Form("Jobtag = {\n \"comment:%s_FinalMerging\"\n};\n", jobTag.Data()));
2117 0 : sjdl2.Prepend("# Generated merging jdl \
2118 : \n# $1 = full alien path to output directory to be merged \
2119 : \n# $2 = merging stage \
2120 : \n# xml made via: find <OutputDir> *Stage<n-1>/*root_archive.zip\n");
2121 : }
2122 0 : index = sjdl1.Index("JDLVariables");
2123 0 : if (index >= 0) sjdl1.Insert(index, "\n# JDL variables\n");
2124 0 : index = sjdl2.Index("JDLVariables");
2125 0 : if (index >= 0) sjdl2.Insert(index, "\n# JDL variables\n");
2126 0 : sjdl1 += "Workdirectorysize = {\"5000MB\"};";
2127 0 : sjdl2 += "Workdirectorysize = {\"5000MB\"};";
2128 0 : index = sjdl2.Index("Split =");
2129 0 : if (index>=0) {
2130 0 : index1 = sjdl2.Index("\n", index);
2131 0 : sjdl2.Remove(index, index1-index+1);
2132 : }
2133 0 : index = sjdl2.Index("SplitMaxInputFileNumber");
2134 0 : if (index>=0) {
2135 0 : index1 = sjdl2.Index("\n", index);
2136 0 : sjdl2.Remove(index, index1-index+1);
2137 : }
2138 0 : index = sjdl2.Index("InputDataCollection");
2139 0 : if (index>=0) {
2140 0 : index1 = sjdl2.Index(";", index);
2141 0 : sjdl2.Remove(index, index1-index+1);
2142 : }
2143 0 : index = sjdl2.Index("InputDataListFormat");
2144 0 : if (index>=0) {
2145 0 : index1 = sjdl2.Index("\n", index);
2146 0 : sjdl2.Remove(index, index1-index+1);
2147 : }
2148 0 : index = sjdl2.Index("InputDataList");
2149 0 : if (index>=0) {
2150 0 : index1 = sjdl2.Index("\n", index);
2151 0 : sjdl2.Remove(index, index1-index+1);
2152 : }
2153 0 : sjdl2.ReplaceAll("wn.xml", Form("Stage_%s.xml",stageName.Data()));
2154 : // Write jdl to file
2155 0 : ofstream out;
2156 0 : out.open(fJDLName.Data(), ios::out);
2157 0 : if (out.bad()) {
2158 0 : Error("WriteJDL", "Bad file name: %s", fJDLName.Data());
2159 0 : return kFALSE;
2160 : }
2161 0 : out << sjdl << endl;
2162 0 : out.close();
2163 0 : TString mergeJDLName = fExecutable;
2164 0 : mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
2165 0 : if (fMergeViaJDL) {
2166 0 : ofstream out1;
2167 0 : out1.open(mergeJDLName.Data(), ios::out);
2168 0 : if (out1.bad()) {
2169 0 : Error("WriteJDL", "Bad file name: %s", mergeJDLName.Data());
2170 0 : return kFALSE;
2171 : }
2172 0 : out1 << sjdl1 << endl;
2173 0 : out1.close();
2174 0 : ofstream out2;
2175 0 : TString finalJDL = mergeJDLName;
2176 0 : finalJDL.ReplaceAll(".jdl", "_final.jdl");
2177 0 : out2.open(finalJDL.Data(), ios::out);
2178 0 : if (out2.bad()) {
2179 0 : Error("WriteJDL", "Bad file name: %s", finalJDL.Data());
2180 0 : return kFALSE;
2181 : }
2182 0 : out2 << sjdl2 << endl;
2183 0 : out2.close();
2184 0 : }
2185 :
2186 : // Copy jdl to grid workspace
2187 0 : if (!copy) {
2188 0 : Info("WriteJDL", "\n##### You may want to review jdl:%s and analysis macro:%s before running in <submit> mode", fJDLName.Data(), fAnalysisMacro.Data());
2189 : } else {
2190 0 : TString locjdl = Form("%s/%s", fGridOutputDir.Data(),fJDLName.Data());
2191 0 : TString locjdl1 = Form("%s/%s", fGridOutputDir.Data(),mergeJDLName.Data());
2192 0 : TString finalJDL = mergeJDLName;
2193 0 : finalJDL.ReplaceAll(".jdl", "_final.jdl");
2194 0 : TString locjdl2 = Form("%s/%s", fGridOutputDir.Data(),finalJDL.Data());
2195 0 : if (fProductionMode) {
2196 0 : locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
2197 0 : locjdl1 = Form("%s/%s", workdir.Data(),mergeJDLName.Data());
2198 0 : locjdl2 = Form("%s/%s", workdir.Data(),finalJDL.Data());
2199 : }
2200 0 : if (FileExists(locjdl)) gGrid->Rm(locjdl);
2201 0 : if (FileExists(locjdl1)) gGrid->Rm(locjdl1);
2202 0 : if (FileExists(locjdl2)) gGrid->Rm(locjdl2);
2203 0 : Info("WriteJDL", "\n##### Copying JDL file <%s> to your AliEn output directory", fJDLName.Data());
2204 : // TFile::Cp(Form("file:%s",fJDLName.Data()), Form("alien://%s", locjdl.Data()));
2205 0 : if (!copyLocal2Alien("WriteJDL",fJDLName.Data(),locjdl.Data()))
2206 0 : Fatal("","Terminating");
2207 0 : if (fMergeViaJDL) {
2208 0 : Info("WriteJDL", "\n##### Copying merging JDL files <%s> to your AliEn output directory", mergeJDLName.Data());
2209 : // TFile::Cp(Form("file:%s",mergeJDLName.Data()), Form("alien://%s", locjdl1.Data()));
2210 : // TFile::Cp(Form("file:%s",finalJDL.Data()), Form("alien://%s", locjdl2.Data()));
2211 0 : if (!copyLocal2Alien("WriteJDL",mergeJDLName.Data(),locjdl1.Data()))
2212 0 : Fatal("","Terminating");
2213 0 : if (!copyLocal2Alien("WriteJDL",finalJDL.Data(),locjdl2.Data()))
2214 0 : Fatal("","Terminating");
2215 : }
2216 0 : }
2217 0 : return kTRUE;
2218 0 : }
2219 :
2220 : //______________________________________________________________________________
2221 : Bool_t AliAnalysisAlien::FileExists(const char *lfn)
2222 : {
2223 : // Returns true if file exists.
2224 0 : if (!gGrid) return kFALSE;
2225 0 : TString slfn = lfn;
2226 0 : slfn.ReplaceAll("alien://","");
2227 0 : TGridResult *res = gGrid->Ls(slfn);
2228 0 : if (!res) return kFALSE;
2229 0 : TMap *map = dynamic_cast<TMap*>(res->At(0));
2230 0 : if (!map) {
2231 0 : delete res;
2232 0 : return kFALSE;
2233 : }
2234 0 : TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("name"));
2235 0 : if (!objs || !objs->GetString().Length()) {
2236 0 : delete res;
2237 0 : return kFALSE;
2238 : }
2239 0 : delete res;
2240 0 : return kTRUE;
2241 0 : }
2242 :
2243 : //______________________________________________________________________________
2244 : Bool_t AliAnalysisAlien::DirectoryExists(const char *dirname)
2245 : {
2246 : // Returns true if directory exists. Can be also a path.
2247 : // Since there is not API in TAlien, we use the Cd trick:
2248 0 : if (!gGrid) return kFALSE;
2249 : // Backup current path
2250 0 : TString cpath = gGrid->Pwd();
2251 0 : TString command = "cd ";
2252 0 : TString sdir(dirname);
2253 0 : sdir.ReplaceAll("alien://", "");
2254 0 : command += sdir;
2255 0 : TGridResult *res = gGrid->Command(command);
2256 0 : if (!res) {
2257 0 : gGrid->Cd(cpath);
2258 0 : return kFALSE;
2259 : }
2260 0 : TMap *map = (TMap*)res->At(0);
2261 0 : if (!map) {
2262 0 : gGrid->Cd(cpath);
2263 0 : delete res;
2264 0 : return kFALSE;
2265 : }
2266 0 : TString sval = map->GetValue("__result__")->GetName();
2267 0 : Bool_t retval = (Bool_t)sval.Atoi();
2268 0 : gGrid->Cd(cpath);
2269 0 : delete res;
2270 0 : return retval;
2271 0 : }
2272 :
2273 : //______________________________________________________________________________
2274 : void AliAnalysisAlien::CheckDataType(const char *lfn, Bool_t &isCollection, Bool_t &isXml, Bool_t &useTags)
2275 : {
2276 : // Check input data type.
2277 0 : isCollection = kFALSE;
2278 0 : isXml = kFALSE;
2279 0 : useTags = kFALSE;
2280 0 : if (!gGrid) {
2281 0 : Error("CheckDataType", "No connection to grid");
2282 0 : return;
2283 : }
2284 0 : isCollection = IsCollection(lfn);
2285 0 : TString msg = "\n##### file: ";
2286 0 : msg += lfn;
2287 0 : if (isCollection) {
2288 0 : msg += " type: raw_collection;";
2289 : // special treatment for collections
2290 0 : isXml = kFALSE;
2291 : // check for tag files in the collection
2292 0 : TGridResult *res = gGrid->Command(Form("listFilesFromCollection -z -v %s",lfn), kFALSE);
2293 0 : if (!res) {
2294 0 : msg += " using_tags: No (unknown)";
2295 0 : Info("CheckDataType", "%s", msg.Data());
2296 0 : return;
2297 : }
2298 0 : const char* typeStr = res->GetKey(0, "origLFN");
2299 0 : if (!typeStr || !strlen(typeStr)) {
2300 0 : msg += " using_tags: No (unknown)";
2301 0 : Info("CheckDataType", "%s", msg.Data());
2302 0 : return;
2303 : }
2304 0 : TString file = typeStr;
2305 0 : useTags = file.Contains(".tag");
2306 0 : if (useTags) msg += " using_tags: Yes";
2307 0 : else msg += " using_tags: No";
2308 0 : Info("CheckDataType", "%s", msg.Data());
2309 : return;
2310 0 : }
2311 0 : TString slfn(lfn);
2312 0 : slfn.ToLower();
2313 0 : isXml = slfn.Contains(".xml");
2314 0 : if (isXml) {
2315 : // Open xml collection and check if there are tag files inside
2316 0 : msg += " type: xml_collection;";
2317 0 : TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"alien://%s\",1);",lfn));
2318 0 : if (!coll) {
2319 0 : msg += " using_tags: No (unknown)";
2320 0 : Info("CheckDataType", "%s", msg.Data());
2321 0 : return;
2322 : }
2323 0 : TMap *map = coll->Next();
2324 0 : if (!map) {
2325 0 : msg += " using_tags: No (unknown)";
2326 0 : Info("CheckDataType", "%s", msg.Data());
2327 0 : return;
2328 : }
2329 0 : map = (TMap*)map->GetValue("");
2330 0 : TString file;
2331 0 : if (map && map->GetValue("name")) file = map->GetValue("name")->GetName();
2332 0 : useTags = file.Contains(".tag");
2333 0 : delete coll;
2334 0 : if (useTags) msg += " using_tags: Yes";
2335 0 : else msg += " using_tags: No";
2336 0 : Info("CheckDataType", "%s", msg.Data());
2337 : return;
2338 0 : }
2339 0 : useTags = slfn.Contains(".tag");
2340 0 : if (slfn.Contains(".root")) msg += " type: root file;";
2341 0 : else msg += " type: unknown file;";
2342 0 : if (useTags) msg += " using_tags: Yes";
2343 0 : else msg += " using_tags: No";
2344 0 : Info("CheckDataType", "%s", msg.Data());
2345 0 : }
2346 :
2347 : //______________________________________________________________________________
2348 : void AliAnalysisAlien::EnablePackage(const char *package)
2349 : {
2350 : // Enables a par file supposed to exist in the current directory.
2351 0 : TString pkg(package);
2352 0 : pkg.ReplaceAll(".par", "");
2353 0 : pkg += ".par";
2354 0 : if (gSystem->AccessPathName(pkg)) {
2355 0 : Fatal("EnablePackage", "Package %s not found", pkg.Data());
2356 0 : return;
2357 : }
2358 0 : if (!TObject::TestBit(AliAnalysisGrid::kUsePars))
2359 0 : Info("EnablePackage", "AliEn plugin will use .par packages");
2360 0 : TObject::SetBit(AliAnalysisGrid::kUsePars, kTRUE);
2361 0 : if (!fPackages) {
2362 0 : fPackages = new TObjArray();
2363 0 : fPackages->SetOwner();
2364 : }
2365 0 : fPackages->Add(new TObjString(pkg));
2366 0 : }
2367 :
2368 : //______________________________________________________________________________
2369 : TChain *AliAnalysisAlien::GetChainForTestMode(const char *treeName) const
2370 : {
2371 : // Make a tree from files having the location specified in fFileForTestMode.
2372 : // Inspired from JF's CreateESDChain.
2373 0 : if (fFileForTestMode.IsNull()) {
2374 0 : Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
2375 0 : return NULL;
2376 : }
2377 0 : if (gSystem->AccessPathName(fFileForTestMode)) {
2378 0 : Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
2379 0 : return NULL;
2380 : }
2381 : // Open the file
2382 0 : ifstream in;
2383 0 : in.open(fFileForTestMode);
2384 : Int_t count = 0;
2385 : // Read the input list of files and add them to the chain
2386 0 : TString line;
2387 0 : TString streeName(treeName);
2388 0 : if (IsUseMCchain()) streeName = "TE";
2389 0 : TChain *chain = new TChain(streeName);
2390 0 : TList *friends = new TList();
2391 : TChain *cfriend = 0;
2392 0 : if (!fFriendChainName.IsNull()) {
2393 0 : TObjArray *list = fFriendChainName.Tokenize(" ");
2394 0 : TIter next(list);
2395 : TObjString *str;
2396 0 : while((str=(TObjString*)next())) {
2397 0 : cfriend = new TChain(streeName, str->GetName());
2398 0 : friends->Add(cfriend);
2399 0 : chain->AddFriend(cfriend);
2400 : }
2401 0 : delete list;
2402 0 : }
2403 0 : TString bpath;
2404 0 : TIter nextfriend(friends);
2405 0 : while (in.good())
2406 : {
2407 0 : in >> line;
2408 0 : if (line.IsNull() || line.BeginsWith("#")) continue;
2409 0 : if (count++ == fNtestFiles) break;
2410 0 : TString esdFile(line);
2411 0 : TFile *file = TFile::Open(esdFile);
2412 0 : if (file && !file->IsZombie()) {
2413 0 : chain->Add(esdFile);
2414 0 : file->Close();
2415 0 : if (!fFriendChainName.IsNull()) {
2416 0 : if (esdFile.Index("#") > -1)
2417 0 : esdFile.Remove(esdFile.Index("#"));
2418 0 : bpath = gSystem->DirName(esdFile);
2419 0 : bpath += "/";
2420 0 : TString fileFriend;
2421 0 : nextfriend.Reset();
2422 0 : while ((cfriend=(TChain*)nextfriend())) {
2423 0 : fileFriend = bpath;
2424 0 : fileFriend += cfriend->GetTitle();
2425 0 : file = TFile::Open(fileFriend);
2426 0 : if (file && !file->IsZombie()) {
2427 0 : file->Close();
2428 0 : cfriend->Add(fileFriend);
2429 : } else {
2430 0 : Fatal("GetChainForTestMode", "Cannot open friend file: %s", fileFriend.Data());
2431 0 : return 0;
2432 : }
2433 : }
2434 0 : }
2435 : } else {
2436 0 : Error("GetChainforTestMode", "Skipping un-openable file: %s", esdFile.Data());
2437 : }
2438 0 : }
2439 0 : in.close();
2440 0 : if (!chain->GetListOfFiles()->GetEntries()) {
2441 0 : Error("GetChainForTestMode", "No file from %s could be opened", fFileForTestMode.Data());
2442 0 : delete chain;
2443 0 : friends->Delete();
2444 0 : delete friends;
2445 0 : return NULL;
2446 : }
2447 0 : return chain;
2448 0 : }
2449 :
2450 : //______________________________________________________________________________
2451 : const char *AliAnalysisAlien::GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone)
2452 : {
2453 : // Get job status for all jobs with jobid>jobidstart.
2454 : static char mstatus[20];
2455 0 : mstatus[0] = '\0';
2456 0 : nrunning = 0;
2457 0 : nwaiting = 0;
2458 0 : nerror = 0;
2459 0 : ndone = 0;
2460 0 : TGridJobStatusList *list = gGrid->Ps("");
2461 0 : if (!list) return mstatus;
2462 0 : Int_t nentries = list->GetSize();
2463 : TGridJobStatus *status;
2464 : Int_t pid;
2465 0 : for (Int_t ijob=0; ijob<nentries; ijob++) {
2466 0 : status = (TGridJobStatus *)list->At(ijob);
2467 0 : pid = gROOT->ProcessLine(Form("atoi(((TAlienJobStatus*)%p)->GetKey(\"queueId\"));", status));
2468 0 : if (pid<jobidstart) continue;
2469 0 : if (pid == lastid) {
2470 0 : gROOT->ProcessLine(Form("sprintf((char*)%p,((TAlienJobStatus*)%p)->GetKey(\"status\"));",mstatus, status));
2471 0 : }
2472 0 : switch (status->GetStatus()) {
2473 : case TGridJobStatus::kWAITING:
2474 0 : nwaiting++; break;
2475 : case TGridJobStatus::kRUNNING:
2476 0 : nrunning++; break;
2477 : case TGridJobStatus::kABORTED:
2478 : case TGridJobStatus::kFAIL:
2479 : case TGridJobStatus::kUNKNOWN:
2480 0 : nerror++; break;
2481 : case TGridJobStatus::kDONE:
2482 0 : ndone++;
2483 0 : }
2484 : }
2485 0 : list->Delete();
2486 0 : delete list;
2487 : return mstatus;
2488 0 : }
2489 :
2490 : //______________________________________________________________________________
2491 : Bool_t AliAnalysisAlien::IsCollection(const char *lfn) const
2492 : {
2493 : // Returns true if file is a collection. Functionality duplicated from
2494 : // TAlien::Type() because we don't want to directly depend on TAlien.
2495 0 : if (!gGrid) {
2496 0 : Error("IsCollection", "No connection to grid");
2497 0 : return kFALSE;
2498 : }
2499 0 : TGridResult *res = gGrid->Command(Form("type -z %s",lfn),kFALSE);
2500 0 : if (!res) return kFALSE;
2501 0 : const char* typeStr = res->GetKey(0, "type");
2502 0 : if (!typeStr || !strlen(typeStr)) return kFALSE;
2503 0 : if (!strcmp(typeStr, "collection")) return kTRUE;
2504 0 : delete res;
2505 0 : return kFALSE;
2506 0 : }
2507 :
2508 : //______________________________________________________________________________
2509 : Bool_t AliAnalysisAlien::IsSingleOutput() const
2510 : {
2511 : // Check if single-ouput option is on.
2512 0 : return (!fOutputSingle.IsNull());
2513 : }
2514 :
2515 : //______________________________________________________________________________
2516 : Long64_t AliAnalysisAlien::RunMacroAndExtractLibs(const char* macro, const char *args, TString &libs)
2517 : {
2518 : // Tries to run the specified macro and return the libraries that it loads.
2519 0 : TString expname;
2520 0 : if (strlen(macro)) expname = gSystem->ExpandPathName(macro);
2521 0 : if (expname.IsNull() || gSystem->AccessPathName(expname)) {
2522 0 : ::Error("RunMacroAndExtractLibs","Cannot find macro %s in current directory", macro);
2523 0 : return -1;
2524 : }
2525 0 : TString oldlibs = gSystem->GetLibraries();
2526 0 : TMacro m(expname);
2527 0 : Int_t error = 0;
2528 0 : Long64_t retval = m.Exec(args, &error);
2529 0 : if (error != TInterpreter::kNoError)
2530 : {
2531 0 : ::Error("RunMacroAndExtractLibs", "Macro interpretation %s failed", macro);
2532 0 : return -1;
2533 : }
2534 0 : libs = gSystem->GetLibraries();
2535 0 : libs.ReplaceAll(oldlibs, "");
2536 0 : libs.Strip(TString::kLeading);
2537 0 : TObjArray *libTokens = libs.Tokenize(" ");
2538 0 : libs = "";
2539 0 : for (Int_t i=0; i<libTokens->GetEntries(); i++) {
2540 0 : if (!libs.IsNull()) libs += " ";
2541 0 : libs += gSystem->BaseName(libTokens->At(i)->GetName());
2542 : }
2543 0 : delete libTokens;
2544 : return retval;
2545 0 : }
2546 :
2547 : //______________________________________________________________________________
2548 : void AliAnalysisAlien::Print(Option_t *) const
2549 : {
2550 : // Print current plugin settings.
2551 0 : printf("### AliEn analysis plugin current settings ###\n");
2552 0 : AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
2553 0 : if (mgr && mgr->IsProofMode()) {
2554 0 : TString proofType = "= PLUGIN IN PROOF MODE ON CLUSTER:_________________";
2555 0 : if (TestBit(AliAnalysisGrid::kTest))
2556 0 : proofType = "= PLUGIN IN PROOF LITE MODE ON CLUSTER:____________";
2557 0 : printf("%s %s\n", proofType.Data(), fProofCluster.Data());
2558 0 : if (!fProofDataSet.IsNull())
2559 0 : printf("= Requested data set:___________________________ %s\n", fProofDataSet.Data());
2560 0 : if (fProofReset==1)
2561 0 : printf("= Soft reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2562 0 : if (fProofReset>1)
2563 0 : printf("= Hard reset signal will be send to master______ CHANGE BEHAVIOR AFTER COMPLETION\n");
2564 0 : if (!fROOTVersion.IsNull())
2565 0 : printf("= ROOT version requested________________________ %s\n", fROOTVersion.Data());
2566 : else
2567 0 : printf("= ROOT version requested________________________ default\n");
2568 0 : printf("= AliRoot version requested_____________________ %s\n", fAliROOTVersion.Data());
2569 0 : printf("= AliPhysics version requested__________________ %s\n", fAliPhysicsVersion.Data());
2570 0 : if (!fAliRootMode.IsNull())
2571 0 : printf("= Requested AliRoot mode________________________ %s\n", fAliRootMode.Data());
2572 0 : if (fNproofWorkers)
2573 0 : printf("= Number of PROOF workers limited to____________ %d\n", fNproofWorkers);
2574 0 : if (fNproofWorkersPerSlave)
2575 0 : printf("= Maximum number of workers per slave___________ %d\n", fNproofWorkersPerSlave);
2576 0 : if (TestSpecialBit(kClearPackages))
2577 0 : printf("= ClearPackages requested...\n");
2578 0 : if (fIncludePath.Data())
2579 0 : printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2580 0 : printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2581 0 : if (fPackages && fPackages->GetEntries()) {
2582 0 : TIter next(fPackages);
2583 : TObject *obj;
2584 0 : TString list;
2585 0 : while ((obj=next())) list += obj->GetName();
2586 0 : printf("= Par files to be used: ________________________ %s\n", list.Data());
2587 0 : }
2588 0 : if (TestSpecialBit(kProofConnectGrid))
2589 0 : printf("= Requested PROOF connection to grid\n");
2590 : return;
2591 0 : }
2592 0 : printf("= OverwriteMode:________________________________ %d\n", fOverwriteMode);
2593 0 : if (fOverwriteMode) {
2594 0 : printf("***** NOTE: Overwrite mode will overwrite the input generated datasets and partial results from previous analysis. \
2595 : \n***** To disable, use: plugin->SetOverwriteMode(kFALSE);\n");
2596 0 : }
2597 0 : printf("= Copy files to grid: __________________________ %s\n", (IsUseCopy())?"YES":"NO");
2598 0 : printf("= Check if files can be copied to grid: ________ %s\n", (IsCheckCopy())?"YES":"NO:Print");
2599 0 : printf("= Production mode:______________________________ %d\n", fProductionMode);
2600 0 : printf("= Version of API requested: ____________________ %s\n", fAPIVersion.Data());
2601 0 : printf("= Version of ROOT requested: ___________________ %s\n", fROOTVersion.Data());
2602 0 : printf("= Version of AliRoot requested: ________________ %s\n", fAliROOTVersion.Data());
2603 0 : printf("= Version of AliPhysics requested: _____________ %s\n", fAliPhysicsVersion.Data());
2604 0 : if (fUser.Length())
2605 0 : printf("= User running the plugin: _____________________ %s\n", fUser.Data());
2606 0 : printf("= Grid workdir relative to user $HOME: _________ %s\n", fGridWorkingDir.Data());
2607 0 : printf("= Grid output directory relative to workdir: ___ %s\n", fGridOutputDir.Data());
2608 0 : TString basedatadir = fGridDataDir;
2609 0 : TString pattern = fDataPattern;
2610 0 : pattern.Strip();
2611 0 : Int_t ind = pattern.Index(" ");
2612 0 : if (ind>=0) {
2613 0 : basedatadir += "/%run%/";
2614 0 : basedatadir += pattern(0, ind);
2615 0 : pattern = pattern(ind+1, pattern.Length());
2616 : }
2617 0 : printf("= Data base directory path requested: __________ %s\n", basedatadir.Data());
2618 0 : printf("= Data search pattern: _________________________ %s\n", pattern.Data());
2619 0 : printf("= Input data format: ___________________________ %s\n", fInputFormat.Data());
2620 0 : if (fRunNumbers.Length())
2621 0 : printf("= Run numbers to be processed: _________________ %s\n", fRunNumbers.Data());
2622 0 : if (fRunRange[0])
2623 0 : printf("= Run range to be processed: ___________________ %d-%d\n", fRunRange[0], fRunRange[1]);
2624 0 : if (!fRunRange[0] && !fRunNumbers.Length()) {
2625 0 : TIter next(fInputFiles);
2626 : TObject *obj;
2627 0 : TString list;
2628 0 : while ((obj=next())) list += obj->GetName();
2629 0 : printf("= Input files to be processed: _________________ %s\n", list.Data());
2630 0 : }
2631 0 : if (TestBit(AliAnalysisGrid::kTest))
2632 0 : printf("= Number of input files used in test mode: _____ %d\n", fNtestFiles);
2633 0 : printf("= List of output files to be registered: _______ %s\n", fOutputFiles.Data());
2634 0 : printf("= List of outputs going to be archived: ________ %s\n", fOutputArchive.Data());
2635 0 : printf("= List of outputs that should not be merged: ___ %s\n", fMergeExcludes.Data());
2636 0 : printf("= List of outputs that should not be registered: %s\n", fRegisterExcludes.Data());
2637 0 : printf("= List of outputs produced during Terminate: ___ %s\n", fTerminateFiles.Data());
2638 0 : printf("=====================================================================\n");
2639 0 : printf("= Job price: ___________________________________ %d\n", fPrice);
2640 0 : printf("= Time to live (TTL): __________________________ %d\n", fTTL);
2641 0 : printf("= Max files per subjob: ________________________ %d\n", fSplitMaxInputFileNumber);
2642 0 : if (fMaxInitFailed>0)
2643 0 : printf("= Max number of subjob fails to kill: __________ %d\n", fMaxInitFailed);
2644 0 : if (fMasterResubmitThreshold>0)
2645 0 : printf("= Resubmit master job if failed subjobs >_______ %d\n", fMasterResubmitThreshold);
2646 0 : printf("= Number of replicas for the output files_______ %d\n", fNreplicas);
2647 0 : if (fNrunsPerMaster>0)
2648 0 : printf("= Number of runs per master job: _______________ %d\n", fNrunsPerMaster);
2649 0 : printf("= Number of files in one chunk to be merged: ___ %d\n", fMaxMergeFiles);
2650 0 : printf("= Name of the generated execution script: ______ %s\n", fExecutable.Data());
2651 0 : printf("= Executable command: __________________________ %s\n", fExecutableCommand.Data());
2652 0 : if (fArguments.Length())
2653 0 : printf("= Arguments for the execution script: __________ %s\n",fArguments.Data());
2654 0 : if (fExecutableArgs.Length())
2655 0 : printf("= Arguments after macro name in executable______ %s\n",fExecutableArgs.Data());
2656 0 : printf("= Name of the generated analysis macro: ________ %s\n",fAnalysisMacro.Data());
2657 0 : printf("= User analysis files to be deployed: __________ %s\n",fAnalysisSource.Data());
2658 0 : printf("= Additional libs to be loaded or souces to be compiled runtime: <%s>\n",fAdditionalLibs.Data());
2659 0 : printf("= Master jobs split mode: ______________________ %s\n",fSplitMode.Data());
2660 0 : if (fDatasetName)
2661 0 : printf("= Custom name for the dataset to be created: ___ %s\n", fDatasetName.Data());
2662 0 : printf("= Name of the generated JDL: ___________________ %s\n", fJDLName.Data());
2663 0 : if (fIncludePath.Data())
2664 0 : printf("= Include path for runtime task compilation: ___ %s\n", fIncludePath.Data());
2665 0 : if (fCloseSE.Length())
2666 0 : printf("= Force job outputs to storage element: ________ %s\n", fCloseSE.Data());
2667 0 : if (fFriendChainName.Length())
2668 0 : printf("= Open friend chain file on worker: ____________ %s\n", fFriendChainName.Data());
2669 0 : if (fPackages && fPackages->GetEntries()) {
2670 0 : TIter next(fPackages);
2671 : TObject *obj;
2672 0 : TString list;
2673 0 : while ((obj=next())) list += obj->GetName();
2674 0 : printf("= Par files to be used: ________________________ %s\n", list.Data());
2675 0 : }
2676 0 : }
2677 :
2678 : //______________________________________________________________________________
2679 : void AliAnalysisAlien::SetDefaults()
2680 : {
2681 : // Set default values for everything. What cannot be filled will be left empty.
2682 0 : if (fGridJDL) delete fGridJDL;
2683 0 : fGridJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
2684 0 : fMergingJDL = (TGridJDL*)gROOT->ProcessLine("new TAlienJDL()");
2685 0 : fPrice = 1;
2686 0 : fTTL = 30000;
2687 0 : fSplitMaxInputFileNumber = 100;
2688 0 : fMaxInitFailed = 0;
2689 0 : fMasterResubmitThreshold = 0;
2690 0 : fNtestFiles = 10;
2691 0 : fNreplicas = 2;
2692 0 : fRunRange[0] = 0;
2693 0 : fRunRange[1] = 0;
2694 0 : fRunPrefix = "%d";
2695 0 : fNrunsPerMaster = 1;
2696 0 : fMaxMergeFiles = 100;
2697 0 : fRunNumbers = "";
2698 0 : fExecutable = "analysis.sh";
2699 0 : fExecutableCommand = "root -b -q -x";
2700 0 : fArguments = "";
2701 0 : fExecutableArgs = "";
2702 0 : fAnalysisMacro = "myAnalysis.C";
2703 0 : fAnalysisSource = "";
2704 0 : fAdditionalLibs = "";
2705 0 : fSplitMode = "se";
2706 0 : fAPIVersion = "";
2707 0 : fROOTVersion = "";
2708 0 : fAliROOTVersion = "";
2709 0 : fAliPhysicsVersion = "";
2710 0 : fUser = ""; // Your alien user name
2711 0 : fGridWorkingDir = "";
2712 0 : fGridDataDir = ""; // Can be like: /alice/sim/PDC_08a/LHC08c9/
2713 0 : fDataPattern = "*AliESDs.root"; // Can be like: *AliESDs.root, */pass1/*AliESDs.root, ...
2714 0 : fFriendChainName = "";
2715 0 : fGridOutputDir = "output";
2716 0 : fOutputArchive = "log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2";
2717 0 : fOutputFiles = ""; // Like "AliAODs.root histos.root"
2718 0 : fInputFormat = "xml-single";
2719 0 : fJDLName = "analysis.jdl";
2720 0 : fJobTag = "Automatically generated analysis JDL";
2721 0 : fMergeExcludes = "";
2722 0 : fMergeViaJDL = 0;
2723 0 : SetUseCopy(kTRUE);
2724 0 : SetCheckCopy(kTRUE);
2725 0 : SetDefaultOutputs(kTRUE);
2726 0 : fOverwriteMode = 1;
2727 0 : }
2728 :
2729 : //______________________________________________________________________________
2730 : void AliAnalysisAlien::SetFriendChainName(const char *name, const char *libnames)
2731 : {
2732 : // Set file name for the chain of friends and optionally additional libs to be loaded.
2733 : // Libs should be separated by blancs.
2734 0 : fFriendChainName = name;
2735 0 : fFriendChainName.ReplaceAll(",", " ");
2736 0 : fFriendChainName.Strip();
2737 0 : fFriendChainName.ReplaceAll(" ", " ");
2738 :
2739 0 : fFriendLibs = libnames;
2740 0 : if (fFriendLibs.Length()) {
2741 0 : if(!fFriendLibs.Contains(".so") &&
2742 0 : !fFriendLibs.Contains(".dylib"))
2743 0 : Fatal("SetFriendChainName()", "You should provide explicit library names (with extension)");
2744 0 : fFriendLibs.ReplaceAll(",", " ");
2745 0 : fFriendLibs.Strip();
2746 0 : fFriendLibs.ReplaceAll(" ", " ");
2747 0 : }
2748 0 : }
2749 :
2750 : //______________________________________________________________________________
2751 : void AliAnalysisAlien::SetRootVersionForProof(const char *version)
2752 : {
2753 : // Obsolete method. Use SetROOTVersion instead
2754 0 : Warning("SetRootVersionForProof", "Obsolete. Use SetROOTVersion instead");
2755 0 : if (fROOTVersion.IsNull()) SetROOTVersion(version);
2756 0 : else Error("SetRootVersionForProof", "ROOT version already set to %s", fROOTVersion.Data());
2757 0 : }
2758 :
2759 : //______________________________________________________________________________
2760 : Bool_t AliAnalysisAlien::CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl)
2761 : {
2762 : // Checks current merge stage, makes xml for the next stage, counts number of files, submits next stage.
2763 : // First check if the result is already in the output directory.
2764 0 : if (FileExists(Form("%s/%s",aliendir,filename))) {
2765 0 : printf("Final merged results found. Not merging again.\n");
2766 0 : return kFALSE;
2767 : }
2768 : // Now check the last stage done.
2769 : Int_t stage = 0;
2770 0 : while (1) {
2771 0 : if (!FileExists(Form("%s/Stage_%d.xml",aliendir, stage+1))) break;
2772 : stage++;
2773 : }
2774 : // Next stage of merging
2775 : stage++;
2776 0 : TString pattern = "*root_archive.zip";
2777 0 : if (stage>1) pattern = Form("Stage_%d/*root_archive.zip", stage-1);
2778 0 : TGridResult *res = gGrid->Command(Form("find -x Stage_%d %s %s", stage, aliendir, pattern.Data()));
2779 0 : if (res) delete res;
2780 : // Write standard output to file
2781 0 : gROOT->ProcessLine(Form("gGrid->Stdout(); > %s", Form("Stage_%d.xml",stage)));
2782 : // Count the number of files inside
2783 0 : ifstream ifile;
2784 0 : ifile.open(Form("Stage_%d.xml",stage));
2785 0 : if (!ifile.good()) {
2786 0 : ::Error("CheckMergedFiles", "Could not redirect result of the find command to file %s", Form("Stage_%d.xml",stage));
2787 0 : return kFALSE;
2788 : }
2789 0 : TString line;
2790 : Int_t nfiles = 0;
2791 0 : while (!ifile.eof()) {
2792 0 : ifile >> line;
2793 0 : if (line.Contains("/event")) nfiles++;
2794 : }
2795 0 : ifile.close();
2796 0 : if (!nfiles) {
2797 0 : ::Error("CheckMergedFiles", "Cannot start Stage_%d merging since Stage_%d did not produced yet output", stage, stage-1);
2798 0 : return kFALSE;
2799 : } else {
2800 0 : printf("=== Stage_%d produced %d files\n", stage-1, nfiles);
2801 : }
2802 : // Copy the file in the output directory
2803 0 : printf("===> Copying collection %s in the output directory %s\n", Form("Stage_%d.xml",stage), aliendir);
2804 : // TFile::Cp(Form("Stage_%d.xml",stage), Form("alien://%s/Stage_%d.xml",aliendir,stage));
2805 0 : if (!copyLocal2Alien("CheckMergedFiles", Form("Stage_%d.xml",stage),
2806 0 : Form("%s/Stage_%d.xml",aliendir,stage))) Fatal("","Terminating");
2807 : // Check if this is the last stage to be done.
2808 0 : Bool_t laststage = (nfiles<nperchunk);
2809 0 : if (fMaxMergeStages && stage>=fMaxMergeStages) laststage = kTRUE;
2810 : Int_t jobId = 0;
2811 0 : if (laststage) {
2812 0 : printf("### Submiting final merging stage %d\n", stage);
2813 0 : TString finalJDL = jdl;
2814 0 : finalJDL.ReplaceAll(".jdl", "_final.jdl");
2815 0 : TString query = Form("submit %s %s %d", finalJDL.Data(), aliendir, stage);
2816 0 : jobId = SubmitSingleJob(query);
2817 0 : } else {
2818 0 : printf("### Submiting merging stage %d\n", stage);
2819 0 : TString query = Form("submit %s %s %d", jdl, aliendir, stage);
2820 0 : jobId = SubmitSingleJob(query);
2821 0 : }
2822 0 : if (!jobId) return kFALSE;
2823 :
2824 0 : if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
2825 0 : fGridJobIDs.Append(Form("%d", jobId));
2826 0 : if (!fGridStages.IsNull()) fGridStages.Append(" ");
2827 0 : fGridStages.Append(Form("%s_merge_stage%d",
2828 0 : laststage ? "final" : "partial", stage));
2829 :
2830 0 : return kTRUE;
2831 0 : }
2832 :
2833 : //______________________________________________________________________________
2834 : AliAnalysisManager *AliAnalysisAlien::LoadAnalysisManager(const char *fname)
2835 : {
2836 : // Loat the analysis manager from a file.
2837 0 : TFile *file = TFile::Open(fname);
2838 0 : if (!file) {
2839 0 : ::Error("LoadAnalysisManager", "Cannot open file %s", fname);
2840 0 : return 0;
2841 : }
2842 0 : TIter nextkey(file->GetListOfKeys());
2843 : AliAnalysisManager *mgr = 0;
2844 : TKey *key;
2845 0 : while ((key=(TKey*)nextkey())) {
2846 0 : if (!strcmp(key->GetClassName(), "AliAnalysisManager"))
2847 0 : mgr = (AliAnalysisManager*)file->Get(key->GetName());
2848 : }
2849 0 : if (!mgr)
2850 0 : ::Error("LoadAnalysisManager", "No analysis manager found in file %s", fname);
2851 : return mgr;
2852 0 : }
2853 :
2854 : //______________________________________________________________________________
2855 : Int_t AliAnalysisAlien::SubmitSingleJob(const char *query)
2856 : {
2857 : // Submits a single job corresponding to the query and returns job id. If 0 submission failed.
2858 0 : if (!gGrid) return 0;
2859 0 : printf("=> %s ------> ",query);
2860 0 : TGridResult *res = gGrid->Command(query);
2861 0 : if (!res) return 0;
2862 0 : TString jobId = res->GetKey(0,"jobId");
2863 0 : delete res;
2864 0 : if (jobId.IsNull()) {
2865 0 : printf("submission failed. Reason:\n");
2866 0 : gGrid->Stdout();
2867 0 : gGrid->Stderr();
2868 0 : ::Error("SubmitSingleJob", "Your query %s could not be submitted", query);
2869 0 : return 0;
2870 : }
2871 0 : Int_t ijobId = jobId.Atoi();
2872 0 : printf(" Job id: '%s' (%d)\n", jobId.Data(), ijobId);
2873 : return ijobId;
2874 0 : }
2875 :
2876 : //______________________________________________________________________________
2877 : Bool_t AliAnalysisAlien::MergeInfo(const char *output, const char *collection)
2878 : {
2879 : // Merges a collection of output files using concatenation.
2880 0 : TString scoll(collection);
2881 0 : if (!scoll.Contains(".xml")) return kFALSE;
2882 0 : TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", collection));
2883 0 : if (!coll) {
2884 0 : ::Error("MergeInfo", "Input XML %s collection empty.", collection);
2885 0 : return kFALSE;
2886 : }
2887 : // Iterate grid collection
2888 0 : TString outtmp;
2889 : Bool_t merged = kFALSE;
2890 : Int_t ifile = 0;
2891 0 : while (coll->Next()) {
2892 0 : TString fname = gSystem->DirName(coll->GetTURL());
2893 0 : fname += "/";
2894 0 : fname += output;
2895 0 : outtmp = Form("%d_%s", ifile, output);
2896 0 : if (!TFile::Cp(fname, outtmp)) {
2897 0 : ::Error("MergeInfo", "Could not copy %s", fname.Data());
2898 0 : continue;
2899 : }
2900 0 : ifile++;
2901 0 : if (ifile<2) {
2902 0 : gSystem->Exec(Form("cp %s lastmerged", outtmp.Data()));
2903 0 : continue;
2904 : }
2905 0 : gSystem->Exec(Form("cat lastmerged %s > tempmerged", outtmp.Data()));
2906 0 : gSystem->Exec("cp tempmerged lastmerged");
2907 0 : }
2908 0 : if (ifile) {
2909 0 : gSystem->Exec(Form("cp lastmerged %s", output));
2910 0 : gSystem->Exec(Form("rm tempmerged lastmerged *_%s", output));
2911 : merged = kTRUE;
2912 0 : }
2913 0 : return merged;
2914 0 : }
2915 :
2916 : //______________________________________________________________________________
2917 : Bool_t AliAnalysisAlien::MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage)
2918 : {
2919 : // Merge given output files from basedir. Basedir can be an alien output directory
2920 : // but also an xml file with root_archive.zip locations. The file merger will merge nmaxmerge
2921 : // files in a group (ignored for xml input). Merging can be done in stages:
2922 : // stage=0 : will merge all existing files in a single stage, supporting resume if run locally
2923 : // stage=1 : works with an xml of all root_archive.zip in the output directory
2924 : // stage>1 : works with an xml of all root_archive.zip in the Stage_<n-1> directory
2925 0 : TString outputFile = output;
2926 0 : TString command;
2927 0 : TString outputChunk;
2928 0 : TString previousChunk = "";
2929 0 : TObjArray *listoffiles = new TObjArray();
2930 : // listoffiles->SetOwner();
2931 : Int_t countChunk = 0;
2932 : Int_t countZero = nmaxmerge;
2933 : Bool_t merged = kTRUE;
2934 : Bool_t isGrid = kTRUE;
2935 0 : Int_t index = outputFile.Index("@");
2936 0 : if (index > 0) outputFile.Remove(index);
2937 0 : TString inputFile = outputFile;
2938 0 : TString sbasedir = basedir;
2939 0 : if (sbasedir.Contains(".xml")) {
2940 : // Merge files pointed by the xml - ignore nmaxmerge and set ichunk to 0
2941 : nmaxmerge = 9999999;
2942 0 : TGridCollection *coll = (TGridCollection*)gROOT->ProcessLine(Form("TAlienCollection::Open(\"%s\");", basedir));
2943 0 : if (!coll) {
2944 0 : ::Error("MergeOutput", "Input XML collection empty.");
2945 0 : return kFALSE;
2946 : }
2947 : // Iterate grid collection
2948 0 : while (coll->Next()) {
2949 0 : TString fname = gSystem->DirName(coll->GetTURL());
2950 0 : fname += "/";
2951 0 : fname += inputFile;
2952 0 : listoffiles->Add(new TNamed(fname.Data(),""));
2953 0 : }
2954 0 : } else if (sbasedir.Contains(".txt")) {
2955 : // The file having the .txt extension is expected to contain a list of
2956 : // folders where the output files will be looked. For alien folders,
2957 : // the full folder LFN is expected (starting with alien://)
2958 : // Assume lfn's on each line
2959 0 : TString line;
2960 0 : ifstream in;
2961 0 : in.open(sbasedir);
2962 0 : if (in.fail()) {
2963 0 : ::Error("MergeOutput", "File %s cannot be opened. Merging stopped." ,sbasedir.Data());
2964 0 : return kTRUE;
2965 : }
2966 : Int_t nfiles = 0;
2967 0 : while (in.good()) {
2968 0 : in >> line;
2969 0 : if (line.IsNull() || line.BeginsWith("#")) continue;
2970 0 : line.Strip();
2971 0 : if (!line.Contains("alien:")) isGrid = kFALSE;
2972 0 : line += "/";
2973 0 : line += outputFile;
2974 0 : nfiles++;
2975 0 : listoffiles->Add(new TNamed(line.Data(),""));
2976 : }
2977 0 : in.close();
2978 0 : if (!nfiles) {
2979 0 : ::Error("MergeOutput","Input file %s contains no files to be merged\n", sbasedir.Data());
2980 0 : delete listoffiles;
2981 0 : return kFALSE;
2982 : }
2983 0 : } else {
2984 0 : command = Form("find %s/ *%s", basedir, inputFile.Data());
2985 0 : printf("command: %s\n", command.Data());
2986 0 : TGridResult *res = gGrid->Command(command);
2987 0 : if (!res) {
2988 0 : ::Error("MergeOutput","No result for the find command\n");
2989 0 : delete listoffiles;
2990 0 : return kFALSE;
2991 : }
2992 0 : TIter nextmap(res);
2993 : TMap *map = 0;
2994 0 : while ((map=(TMap*)nextmap())) {
2995 0 : TObjString *objs = dynamic_cast<TObjString*>(map->GetValue("turl"));
2996 0 : if (!objs || !objs->GetString().Length()) {
2997 : // Nothing found - skip this output
2998 0 : delete res;
2999 0 : delete listoffiles;
3000 0 : return kFALSE;
3001 : }
3002 0 : listoffiles->Add(new TNamed(objs->GetName(),""));
3003 0 : }
3004 0 : delete res;
3005 0 : }
3006 0 : if (!listoffiles->GetEntries()) {
3007 0 : ::Error("MergeOutput","No result for the find command\n");
3008 0 : delete listoffiles;
3009 0 : return kFALSE;
3010 : }
3011 :
3012 : TFileMerger *fm = 0;
3013 0 : TIter next0(listoffiles);
3014 0 : TObjArray *listoffilestmp = new TObjArray();
3015 0 : listoffilestmp->SetOwner();
3016 : TObject *nextfile;
3017 0 : TString snextfile;
3018 : // Keep only the files at upper level
3019 : Int_t countChar = 0;
3020 0 : while ((nextfile=next0())) {
3021 0 : snextfile = nextfile->GetName();
3022 0 : Int_t crtCount = snextfile.CountChar('/');
3023 0 : if (nextfile == listoffiles->First()) countChar = crtCount;
3024 0 : if (crtCount < countChar) countChar = crtCount;
3025 : }
3026 0 : next0.Reset();
3027 0 : while ((nextfile=next0())) {
3028 0 : snextfile = nextfile->GetName();
3029 0 : Int_t crtCount = snextfile.CountChar('/');
3030 0 : if (crtCount > countChar) {
3031 0 : delete nextfile;
3032 0 : continue;
3033 : }
3034 0 : listoffilestmp->Add(nextfile);
3035 0 : }
3036 0 : delete listoffiles;
3037 : listoffiles = listoffilestmp; // Now contains 'good' files
3038 0 : listoffiles->Print();
3039 0 : TIter next(listoffiles);
3040 : // Check if there is a merge operation to resume. Works only for stage 0 or 1.
3041 0 : outputChunk = outputFile;
3042 0 : outputChunk.ReplaceAll(".root", "_*.root");
3043 : // Check for existent temporary merge files
3044 : // Check overwrite mode and remove previous partial results if needed
3045 : // Preserve old merging functionality for stage 0.
3046 0 : if (stage==0) {
3047 0 : if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
3048 : while (1) {
3049 : // Skip as many input files as in a chunk
3050 0 : for (Int_t counter=0; counter<nmaxmerge; counter++) {
3051 0 : nextfile = next();
3052 0 : if (!nextfile) {
3053 0 : ::Error("MergeOutput", "Mismatch found. Please remove partial merged files from local dir.");
3054 0 : delete listoffiles;
3055 0 : return kFALSE;
3056 : }
3057 0 : snextfile = nextfile->GetName();
3058 : }
3059 0 : outputChunk = outputFile;
3060 0 : outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
3061 0 : countChunk++;
3062 0 : if (gSystem->AccessPathName(outputChunk)) continue;
3063 : // Merged file with chunks up to <countChunk> found
3064 0 : ::Info("MergeOutput", "Resume merging of <%s> from <%s>\n", outputFile.Data(), outputChunk.Data());
3065 0 : previousChunk = outputChunk;
3066 : break;
3067 : }
3068 : }
3069 : countZero = nmaxmerge;
3070 :
3071 0 : while ((nextfile=next())) {
3072 0 : snextfile = nextfile->GetName();
3073 : // Loop 'find' results and get next LFN
3074 0 : if (countZero == nmaxmerge) {
3075 : // First file in chunk - create file merger and add previous chunk if any.
3076 0 : fm = new TFileMerger(isGrid);
3077 0 : fm->SetFastMethod(kTRUE);
3078 0 : if (previousChunk.Length()) fm->AddFile(previousChunk.Data());
3079 0 : outputChunk = outputFile;
3080 0 : outputChunk.ReplaceAll(".root", Form("_%04d.root", countChunk));
3081 : }
3082 : // If last file found, put merged results in the output file
3083 0 : if (nextfile == listoffiles->Last()) outputChunk = outputFile;
3084 : // Add file to be merged and decrement chunk counter.
3085 0 : fm->AddFile(snextfile);
3086 0 : countZero--;
3087 0 : if (countZero==0 || nextfile == listoffiles->Last()) {
3088 0 : if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
3089 : // Nothing found - skip this output
3090 0 : ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
3091 : merged = kFALSE;
3092 0 : break;
3093 : }
3094 0 : fm->OutputFile(outputChunk);
3095 : // Merge the outputs, then go to next chunk
3096 0 : if (!fm->Merge()) {
3097 0 : ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
3098 : merged = kFALSE;
3099 0 : break;
3100 : } else {
3101 0 : ::Info("MergeOutputs", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputChunk.Data());
3102 0 : gSystem->Unlink(previousChunk);
3103 : }
3104 0 : if (nextfile == listoffiles->Last()) break;
3105 0 : countChunk++;
3106 : countZero = nmaxmerge;
3107 0 : previousChunk = outputChunk;
3108 : }
3109 : }
3110 0 : delete listoffiles;
3111 0 : delete fm;
3112 0 : return merged;
3113 : }
3114 : // Merging stage different than 0.
3115 : // Move to the begining of the requested chunk.
3116 0 : fm = new TFileMerger(isGrid);
3117 0 : fm->SetFastMethod(kTRUE);
3118 0 : while ((nextfile=next())) fm->AddFile(nextfile->GetName());
3119 0 : delete listoffiles;
3120 0 : if (!fm->GetMergeList() || !fm->GetMergeList()->GetSize()) {
3121 : // Nothing found - skip this output
3122 0 : ::Warning("MergeOutput", "No <%s> files found.", inputFile.Data());
3123 0 : delete fm;
3124 0 : return kFALSE;
3125 : }
3126 0 : fm->OutputFile(outputFile);
3127 : // Merge the outputs
3128 0 : if (!fm->Merge()) {
3129 0 : ::Error("MergeOutput", "Could not merge all <%s> files", outputFile.Data());
3130 0 : delete fm;
3131 0 : return kFALSE;
3132 : } else {
3133 0 : ::Info("MergeOutput", "\n##### Merged %d output files to <%s>", fm->GetMergeList()->GetSize(), outputFile.Data());
3134 : }
3135 0 : delete fm;
3136 0 : return kTRUE;
3137 0 : }
3138 :
3139 : //______________________________________________________________________________
3140 : Bool_t AliAnalysisAlien::MergeOutputs()
3141 : {
3142 : // Merge analysis outputs existing in the AliEn space.
3143 0 : if (TestBit(AliAnalysisGrid::kTest)) return kTRUE;
3144 0 : if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3145 0 : if (!Connect()) {
3146 0 : Error("MergeOutputs", "Cannot merge outputs without grid connection. Terminate will NOT be executed");
3147 0 : return kFALSE;
3148 : }
3149 0 : if (fMergeViaJDL) {
3150 0 : if (!TestBit(AliAnalysisGrid::kMerge)) {
3151 0 : Info("MergeOutputs", "### Re-run with <MergeViaJDL> option in terminate mode of the plugin to submit merging jobs ###");
3152 0 : return kFALSE;
3153 : }
3154 0 : if (fProductionMode) {
3155 0 : Info("MergeOutputs", "### Merging will be submitted by LPM manager... ###");
3156 0 : return kFALSE;
3157 : }
3158 0 : Info("MergeOutputs", "Submitting merging JDL");
3159 0 : if (!SubmitMerging()) return kFALSE;
3160 0 : Info("MergeOutputs", "### Re-run with <MergeViaJDL> off to collect results after merging jobs are done ###");
3161 0 : Info("MergeOutputs", "### The Terminate() method is executed by the merging jobs");
3162 0 : return kFALSE;
3163 : }
3164 : // Get the output path
3165 0 : if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
3166 0 : if (!DirectoryExists(fGridOutputDir)) {
3167 0 : Error("MergeOutputs", "Grid output directory %s not found. Terminate() will NOT be executed", fGridOutputDir.Data());
3168 0 : return kFALSE;
3169 : }
3170 0 : if (!fOutputFiles.Length()) {
3171 0 : Error("MergeOutputs", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
3172 0 : return kFALSE;
3173 : }
3174 : // Check if fast read option was requested
3175 0 : Info("MergeOutputs", "Started local merging of output files from: alien://%s \
3176 0 : \n======= overwrite mode = %d", fGridOutputDir.Data(), (Int_t)fOverwriteMode);
3177 0 : if (fFastReadOption) {
3178 0 : Warning("MergeOutputs", "You requested FastRead option. Using xrootd flags to reduce timeouts. This may skip some files that could be accessed ! \
3179 : \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
3180 0 : gEnv->SetValue("XNet.ConnectTimeout",50);
3181 0 : gEnv->SetValue("XNet.RequestTimeout",50);
3182 0 : gEnv->SetValue("XNet.MaxRedirectCount",2);
3183 0 : gEnv->SetValue("XNet.ReconnectTimeout",50);
3184 0 : gEnv->SetValue("XNet.FirstConnectMaxCnt",1);
3185 0 : }
3186 : // Make sure we change the temporary directory
3187 0 : gSystem->Setenv("TMPDIR", gSystem->pwd());
3188 : // Set temporary compilation directory to current one
3189 0 : gSystem->SetBuildDir(gSystem->pwd(), kTRUE);
3190 0 : TObjArray *list = fOutputFiles.Tokenize(",");
3191 0 : TIter next(list);
3192 : TObjString *str;
3193 0 : TString outputFile;
3194 : Bool_t merged = kTRUE;
3195 0 : while((str=(TObjString*)next())) {
3196 0 : outputFile = str->GetString();
3197 0 : Int_t index = outputFile.Index("@");
3198 0 : if (index > 0) outputFile.Remove(index);
3199 0 : TString outputChunk = outputFile;
3200 0 : outputChunk.ReplaceAll(".root", "_*.root");
3201 : // Skip already merged outputs
3202 0 : if (!gSystem->AccessPathName(outputFile)) {
3203 0 : if (fOverwriteMode) {
3204 0 : Info("MergeOutputs", "Overwrite mode. Existing file %s was deleted.", outputFile.Data());
3205 0 : gSystem->Unlink(outputFile);
3206 0 : if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
3207 0 : Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
3208 0 : outputChunk.Data());
3209 0 : gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
3210 : }
3211 : } else {
3212 0 : Info("MergeOutputs", "Output file <%s> found. Not merging again.", outputFile.Data());
3213 0 : continue;
3214 : }
3215 : } else {
3216 0 : if (!gSystem->Exec(Form("ls %s 2>/dev/null", outputChunk.Data()))) {
3217 0 : Info("MergeOutput", "Overwrite mode: partial merged files %s will removed",
3218 0 : outputChunk.Data());
3219 0 : gSystem->Exec(Form("rm -f %s", outputChunk.Data()));
3220 : }
3221 : }
3222 0 : if (fMergeExcludes.Contains(outputFile.Data()) ||
3223 0 : fRegisterExcludes.Contains(outputFile.Data())) continue;
3224 : // Perform a 'find' command in the output directory, looking for registered outputs
3225 0 : merged = MergeOutput(outputFile, fGridOutputDir, fMaxMergeFiles);
3226 0 : if (!merged) {
3227 0 : Error("MergeOutputs", "Terminate() will NOT be executed");
3228 0 : delete list;
3229 0 : return kFALSE;
3230 : }
3231 0 : TFile *fileOpened = (TFile*)gROOT->GetListOfFiles()->FindObject(outputFile);
3232 0 : if (fileOpened) fileOpened->Close();
3233 0 : }
3234 0 : delete list;
3235 0 : return kTRUE;
3236 0 : }
3237 :
3238 : //______________________________________________________________________________
3239 : void AliAnalysisAlien::SetDefaultOutputs(Bool_t flag)
3240 : {
3241 : // Use the output files connected to output containers from the analysis manager
3242 : // rather than the files defined by SetOutputFiles
3243 0 : if (flag && !TObject::TestBit(AliAnalysisGrid::kDefaultOutputs))
3244 0 : Info("SetDefaultOutputs", "Plugin will use the output files taken from analysis manager");
3245 0 : TObject::SetBit(AliAnalysisGrid::kDefaultOutputs, flag);
3246 0 : }
3247 :
3248 : //______________________________________________________________________________
3249 : void AliAnalysisAlien::SetOutputFiles(const char *list)
3250 : {
3251 : // Manually set the output files list.
3252 : // Removes duplicates. Not allowed if default outputs are not disabled.
3253 0 : if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3254 0 : Fatal("SetOutputFiles", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set output files.");
3255 0 : return;
3256 : }
3257 0 : Info("SetOutputFiles", "Output file list is set manually - you are on your own.");
3258 0 : fOutputFiles = "";
3259 0 : TString slist = list;
3260 0 : if (slist.Contains("@")) Warning("SetOutputFiles","The plugin does not allow explicit SE's. Please use: SetNumberOfReplicas() instead.");
3261 0 : TObjArray *arr = slist.Tokenize(" ");
3262 : TObjString *os;
3263 0 : TIter next(arr);
3264 0 : TString sout;
3265 0 : while ((os=(TObjString*)next())) {
3266 0 : sout = os->GetString();
3267 0 : if (sout.Index("@")>0) sout.Remove(sout.Index("@"));
3268 0 : if (fOutputFiles.Contains(sout)) continue;
3269 0 : if (!fOutputFiles.IsNull()) fOutputFiles += ",";
3270 0 : fOutputFiles += sout;
3271 : }
3272 0 : delete arr;
3273 0 : }
3274 :
3275 : //______________________________________________________________________________
3276 : void AliAnalysisAlien::SetOutputArchive(const char *list)
3277 : {
3278 : // Manually set the output archive list. Free text - you are on your own...
3279 : // Not allowed if default outputs are not disabled.
3280 0 : if (TObject::TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3281 0 : Fatal("SetOutputArchive", "You have to explicitly call SetDefaultOutputs(kFALSE) to manually set the output archives.");
3282 0 : return;
3283 : }
3284 0 : Info("SetOutputArchive", "Output archive is set manually - you are on your own.");
3285 0 : fOutputArchive = list;
3286 0 : }
3287 :
3288 : //______________________________________________________________________________
3289 : void AliAnalysisAlien::SetProofParameter(const char *pname, const char *value)
3290 : {
3291 : // Set some PROOF special parameter.
3292 0 : TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3293 0 : if (pair) {
3294 0 : TObject *old = pair->Key();
3295 0 : TObject *val = pair->Value();
3296 0 : fProofParam.Remove(old);
3297 0 : delete old;
3298 0 : delete val;
3299 0 : }
3300 0 : fProofParam.Add(new TObjString(pname), new TObjString(value));
3301 0 : }
3302 :
3303 : //______________________________________________________________________________
3304 : const char *AliAnalysisAlien::GetProofParameter(const char *pname) const
3305 : {
3306 : // Returns a special PROOF parameter.
3307 0 : TPair *pair = dynamic_cast<TPair*>(fProofParam.FindObject(pname));
3308 0 : if (!pair) return 0;
3309 0 : return pair->Value()->GetName();
3310 0 : }
3311 :
3312 : //______________________________________________________________________________
3313 : Bool_t AliAnalysisAlien::StartAnalysis(Long64_t nentries, Long64_t firstEntry)
3314 : {
3315 : // Start remote grid analysis.
3316 0 : AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3317 0 : Bool_t testMode = TestBit(AliAnalysisGrid::kTest);
3318 0 : if (!mgr || !mgr->IsInitialized()) {
3319 0 : Error("StartAnalysis", "You need an initialized analysis manager for this");
3320 0 : return kFALSE;
3321 : }
3322 : // Are we in PROOF mode ?
3323 0 : if (mgr->IsProofMode()) {
3324 0 : if (testMode) Info("StartAnalysis", "##### Starting PROOF analysis with Proof Lite via the plugin #####");
3325 0 : else Info("StartAnalysis", "##### Starting PROOF analysis on cluster <%s> via the plugin #####", fProofCluster.Data());
3326 0 : if (fProofCluster.IsNull()) {
3327 0 : Error("StartAnalysis", "You need to specify the proof cluster name via SetProofCluster");
3328 0 : return kFALSE;
3329 : }
3330 0 : if (fProofDataSet.IsNull() && !testMode) {
3331 0 : Error("StartAnalysis", "You need to specify a dataset using SetProofDataSet()");
3332 0 : return kFALSE;
3333 : }
3334 : // Set the needed environment
3335 0 : gEnv->SetValue("XSec.GSI.DelegProxy","2");
3336 : // Do we need to reset PROOF ? The success of the Reset operation cannot be checked
3337 0 : if (fProofReset && !testMode) {
3338 0 : if (fProofReset==1) {
3339 0 : Info("StartAnalysis", "Sending soft reset signal to proof cluster %s", fProofCluster.Data());
3340 0 : gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kFALSE);", fProofCluster.Data()));
3341 0 : } else {
3342 0 : Info("StartAnalysis", "Sending hard reset signal to proof cluster %s", fProofCluster.Data());
3343 0 : gROOT->ProcessLine(Form("TProof::Reset(\"%s\", kTRUE);", fProofCluster.Data()));
3344 : }
3345 0 : Info("StartAnalysis", "Stopping the analysis. Please use SetProofReset(0) to resume.");
3346 0 : return kFALSE;
3347 : }
3348 :
3349 0 : if (!testMode) {
3350 : // Check if there is an old active session
3351 0 : Long_t nsessions = gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->QuerySessions(\"\")->GetEntries();", fProofCluster.Data()));
3352 0 : if (nsessions) {
3353 0 : Error("StartAnalysis","You have to reset your old session first\n");
3354 0 : return kFALSE;
3355 : }
3356 0 : }
3357 : // Do we need to change the ROOT version ? The success of this cannot be checked.
3358 0 : if (!fROOTVersion.IsNull() && !testMode) {
3359 0 : gROOT->ProcessLine(Form("TProof::Mgr(\"%s\")->SetROOTVersion(\"VO_ALICE@ROOT::%s\");",
3360 0 : fProofCluster.Data(), fROOTVersion.Data()));
3361 0 : }
3362 : // Connect to PROOF and check the status
3363 : Long_t proof = 0;
3364 0 : TString sworkers;
3365 0 : if (fNproofWorkersPerSlave) sworkers = Form("workers=%dx", fNproofWorkersPerSlave);
3366 0 : else if (fNproofWorkers) sworkers = Form("workers=%d", fNproofWorkers);
3367 0 : if (!testMode) {
3368 0 : if (!sworkers.IsNull())
3369 0 : proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\", \"%s\");", fProofCluster.Data(), sworkers.Data()));
3370 : else
3371 0 : proof = gROOT->ProcessLine(Form("TProof::Open(\"%s\");", fProofCluster.Data()));
3372 : } else {
3373 0 : proof = gROOT->ProcessLine("TProof::Open(\"\");");
3374 0 : if (!proof) {
3375 0 : Error("StartAnalysis", "Could not start PROOF in test mode");
3376 0 : return kFALSE;
3377 : }
3378 : }
3379 0 : if (!proof) {
3380 0 : Error("StartAnalysis", "Could not connect to PROOF cluster <%s>", fProofCluster.Data());
3381 0 : return kFALSE;
3382 : }
3383 0 : if (fNproofWorkersPerSlave*fNproofWorkers > 0)
3384 0 : gROOT->ProcessLine(Form("gProof->SetParallel(%d);", fNproofWorkers));
3385 : // Set proof special parameters if any
3386 0 : TIter nextpp(&fProofParam);
3387 : TObject *proofparam;
3388 0 : while ((proofparam=nextpp())) {
3389 0 : TString svalue = GetProofParameter(proofparam->GetName());
3390 0 : gROOT->ProcessLine(Form("gProof->SetParameter(\"%s\",%s);", proofparam->GetName(), svalue.Data()));
3391 0 : }
3392 : // Is dataset existing ?
3393 0 : if (!testMode) {
3394 0 : TString dataset = fProofDataSet;
3395 0 : Int_t index = dataset.Index("#");
3396 0 : if (index>=0) dataset.Remove(index);
3397 : // if (!gROOT->ProcessLine(Form("gProof->ExistsDataSet(\"%s\");",fProofDataSet.Data()))) {
3398 : // Error("StartAnalysis", "Dataset %s not existing", fProofDataSet.Data());
3399 : // return kFALSE;
3400 : // }
3401 : // Info("StartAnalysis", "Dataset %s found", dataset.Data());
3402 0 : }
3403 : // Is ClearPackages() needed ?
3404 0 : if (TestSpecialBit(kClearPackages)) {
3405 0 : Info("StartAnalysis", "ClearPackages signal sent to PROOF. Use SetClearPackages(kFALSE) to reset this.");
3406 0 : gROOT->ProcessLine("gProof->ClearPackages();");
3407 : }
3408 : // Is a given aliroot mode requested ?
3409 0 : TList optionsList;
3410 0 : TString parLibs;
3411 0 : if (!fAliRootMode.IsNull()) {
3412 0 : TString alirootMode = fAliRootMode;
3413 0 : if (alirootMode == "default") alirootMode = "";
3414 0 : Info("StartAnalysis", "You are requesting AliRoot mode: %s", fAliRootMode.Data());
3415 0 : optionsList.SetOwner();
3416 0 : optionsList.Add(new TNamed("ALIROOT_MODE", alirootMode.Data()));
3417 : // Check the additional libs to be loaded
3418 0 : TString extraLibs;
3419 : Bool_t parMode = kFALSE;
3420 0 : if (!alirootMode.IsNull()) extraLibs = "ANALYSIS:ANALYSISalice:OADB";
3421 : // Parse the extra libs for .so or .dylib
3422 0 : if (fAdditionalLibs.Length()) {
3423 0 : TString additionalLibs = fAdditionalLibs;
3424 0 : additionalLibs.Strip();
3425 0 : if (additionalLibs.Length() && fFriendLibs.Length())
3426 0 : additionalLibs += " ";
3427 0 : additionalLibs += fFriendLibs;
3428 0 : TObjArray *list = additionalLibs.Tokenize(" ");
3429 0 : TIter next(list);
3430 : TObjString *str;
3431 0 : while((str=(TObjString*)next())) {
3432 0 : if (str->GetString().Contains(".so") ||
3433 0 : str->GetString().Contains(".dylib") ) {
3434 0 : if (parMode) {
3435 0 : Warning("StartAnalysis", "Plugin does not support loading libs after par files in PROOF mode. Library %s and following will not load on workers", str->GetName());
3436 : break;
3437 : }
3438 0 : TString stmp = str->GetName();
3439 0 : if (stmp.BeginsWith("lib")) stmp.Remove(0,3);
3440 0 : stmp.ReplaceAll(".so","");
3441 0 : stmp.ReplaceAll(".dylib","");
3442 0 : if (!extraLibs.IsNull()) extraLibs += ":";
3443 0 : extraLibs += stmp;
3444 : continue;
3445 0 : }
3446 0 : if (str->GetString().Contains(".par")) {
3447 : // The first par file found in the list will not allow any further .so
3448 : parMode = kTRUE;
3449 0 : if (!parLibs.IsNull()) parLibs += ":";
3450 0 : parLibs += str->GetName();
3451 : continue;
3452 : }
3453 : }
3454 0 : if (list) delete list;
3455 0 : }
3456 0 : if (!extraLibs.IsNull()) {
3457 0 : Info("StartAnalysis", "Adding extra libs: %s",extraLibs.Data());
3458 0 : optionsList.Add(new TNamed("ALIROOT_EXTRA_LIBS",extraLibs.Data()));
3459 : }
3460 : // Check extra includes
3461 0 : if (!fIncludePath.IsNull()) {
3462 0 : TString includePath = fIncludePath;
3463 0 : includePath.ReplaceAll(" ",":");
3464 0 : includePath.ReplaceAll("$ALICE_ROOT/","");
3465 0 : includePath.ReplaceAll("${ALICE_ROOT}/","");
3466 0 : includePath.ReplaceAll("-I","");
3467 0 : includePath.Remove(TString::kTrailing, ':');
3468 0 : Info("StartAnalysis", "Adding extra includes: %s",includePath.Data());
3469 0 : optionsList.Add(new TNamed("ALIROOT_EXTRA_INCLUDES",includePath.Data()));
3470 0 : }
3471 : // Check if connection to grid is requested
3472 0 : if (TestSpecialBit(kProofConnectGrid))
3473 0 : optionsList.Add(new TNamed("ALIROOT_ENABLE_ALIEN", "1"));
3474 : // Enable AliRoot par
3475 0 : if (testMode) {
3476 : // Enable proof lite package
3477 0 : TString alirootLite = gSystem->ExpandPathName("$ALICE_ROOT/ANALYSIS/macros/AliRootProofLite.par");
3478 0 : for (Int_t i=0; i<optionsList.GetSize(); i++) {
3479 0 : TNamed *obj = (TNamed*)optionsList.At(i);
3480 0 : printf("%s %s\n", obj->GetName(), obj->GetTitle());
3481 : }
3482 0 : if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");",alirootLite.Data()))
3483 0 : && !gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\", (TList*)%p);",alirootLite.Data(),&optionsList))) {
3484 0 : Info("StartAnalysis", "AliRootProofLite enabled");
3485 : } else {
3486 0 : Error("StartAnalysis", "There was an error trying to enable package AliRootProofLite.par");
3487 0 : return kFALSE;
3488 : }
3489 0 : } else {
3490 0 : if ( ! fAliROOTVersion.IsNull() ) {
3491 0 : if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliRoot::%s\", (TList*)%p, kTRUE);",
3492 0 : fAliROOTVersion.Data(), &optionsList))) {
3493 0 : Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliRoot::%s", fAliROOTVersion.Data());
3494 0 : return kFALSE;
3495 : }
3496 : }
3497 0 : if ( ! fAliPhysicsVersion.IsNull() ) {
3498 0 : if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"VO_ALICE@AliPhysics::%s\", (TList*)%p, kTRUE);",
3499 0 : fAliPhysicsVersion.Data(), &optionsList))) {
3500 0 : Error("StartAnalysis", "There was an error trying to enable package VO_ALICE@AliPhysics::%s", fAliPhysicsVersion.Data());
3501 0 : return kFALSE;
3502 : }
3503 : }
3504 : }
3505 : // Enable first par files from fAdditionalLibs
3506 0 : if (!parLibs.IsNull()) {
3507 0 : TObjArray *list = parLibs.Tokenize(":");
3508 0 : TIter next(list);
3509 : TObjString *package;
3510 0 : while((package=(TObjString*)next())) {
3511 0 : TString spkg = package->GetName();
3512 0 : spkg.ReplaceAll(".par", "");
3513 0 : gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
3514 0 : if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
3515 0 : TString enablePackage = (testMode)?Form("gProof->EnablePackage(\"%s\",kFALSE);", package->GetName()):Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName());
3516 0 : if (gROOT->ProcessLine(enablePackage)) {
3517 0 : Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3518 0 : return kFALSE;
3519 : }
3520 0 : } else {
3521 0 : Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3522 0 : return kFALSE;
3523 : }
3524 0 : }
3525 0 : if (list) delete list;
3526 0 : }
3527 0 : } else {
3528 0 : if ((fAdditionalLibs.Contains(".so") || fAdditionalLibs.Contains(".dylib")) &&
3529 : !testMode) {
3530 0 : Error("StartAnalysis", "You request additional libs to be loaded but did not enabled any AliRoot mode. Please refer to: \
3531 : \n http://aaf.cern.ch/node/83 and use a parameter for SetAliRootMode()");
3532 0 : return kFALSE;
3533 : }
3534 : }
3535 : // Enable par files if requested
3536 0 : if (fPackages && fPackages->GetEntries()) {
3537 0 : TIter next(fPackages);
3538 : TObject *package;
3539 0 : while ((package=next())) {
3540 : // Skip packages already enabled
3541 0 : if (parLibs.Contains(package->GetName())) continue;
3542 0 : TString spkg = package->GetName();
3543 0 : spkg.ReplaceAll(".par", "");
3544 0 : gSystem->Exec(TString::Format("rm -rf %s", spkg.Data()));
3545 0 : if (!gROOT->ProcessLine(Form("gProof->UploadPackage(\"%s\");", package->GetName()))) {
3546 0 : if (gROOT->ProcessLine(Form("gProof->EnablePackage(\"%s\",kTRUE);", package->GetName()))) {
3547 0 : Error("StartAnalysis", "There was an error trying to enable package %s", package->GetName());
3548 0 : return kFALSE;
3549 : }
3550 : } else {
3551 0 : Error("StartAnalysis", "There was an error trying to upload package %s", package->GetName());
3552 0 : return kFALSE;
3553 : }
3554 0 : }
3555 0 : }
3556 : // Do we need to load analysis source files ?
3557 : // NOTE: don't load on client since this is anyway done by the user to attach his task.
3558 0 : if (fAnalysisSource.Length()) {
3559 0 : TObjArray *list = fAnalysisSource.Tokenize(" ");
3560 0 : TIter next(list);
3561 : TObjString *str;
3562 0 : while((str=(TObjString*)next())) {
3563 0 : gROOT->ProcessLine(Form("gProof->Load(\"%s+g\", kTRUE);", str->GetName()));
3564 : }
3565 0 : if (list) delete list;
3566 0 : }
3567 0 : if (testMode) {
3568 : // Register dataset to proof lite.
3569 0 : if (fFileForTestMode.IsNull()) {
3570 0 : Error("GetChainForTestMode", "For proof test mode please use SetFileForTestMode() pointing to a file that contains data file locations.");
3571 0 : return kFALSE;
3572 : }
3573 0 : if (gSystem->AccessPathName(fFileForTestMode)) {
3574 0 : Error("GetChainForTestMode", "File not found: %s", fFileForTestMode.Data());
3575 0 : return kFALSE;
3576 : }
3577 0 : TFileCollection *coll = new TFileCollection();
3578 0 : coll->AddFromFile(fFileForTestMode);
3579 0 : gROOT->ProcessLine(Form("gProof->RegisterDataSet(\"test_collection\", (TFileCollection*)%p, \"OV\");", coll));
3580 0 : gROOT->ProcessLine("gProof->ShowDataSets()");
3581 0 : }
3582 0 : return kTRUE;
3583 0 : }
3584 :
3585 : // Check if output files have to be taken from the analysis manager
3586 0 : if (TestBit(AliAnalysisGrid::kDefaultOutputs)) {
3587 : // Add output files and AOD files
3588 0 : fOutputFiles = GetListOfFiles("outaod");
3589 : // Add extra files registered to the analysis manager
3590 0 : TString extra = GetListOfFiles("ext");
3591 0 : if (!extra.IsNull()) {
3592 0 : extra.ReplaceAll(".root", "*.root");
3593 0 : if (!fOutputFiles.IsNull()) fOutputFiles += ",";
3594 0 : fOutputFiles += extra;
3595 : }
3596 : // Compose the output archive.
3597 0 : fOutputArchive = "log_archive.zip:std*@disk=1 ";
3598 0 : if (mgr->IsCollectThroughput())
3599 0 : fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d %s@disk=%d",fOutputFiles.Data(),fNreplicas, mgr->GetFileInfoLog(),fNreplicas);
3600 : else
3601 0 : fOutputArchive += Form("root_archive.zip:%s,*.stat@disk=%d",fOutputFiles.Data(),fNreplicas);
3602 0 : }
3603 : // if (!fCloseSE.Length()) fCloseSE = gSystem->Getenv("alien_CLOSE_SE");
3604 0 : if (TestBit(AliAnalysisGrid::kOffline)) {
3605 0 : Info("StartAnalysis","\n##### OFFLINE MODE ##### Files to be used in GRID are produced but not copied \
3606 : \n there nor any job run. You can revise the JDL and analysis \
3607 : \n macro then run the same in \"submit\" mode.");
3608 0 : } else if (TestBit(AliAnalysisGrid::kTest)) {
3609 0 : Info("StartAnalysis","\n##### LOCAL MODE ##### Your analysis will be run locally on a subset of the requested \
3610 : \n dataset.");
3611 0 : } else if (TestBit(AliAnalysisGrid::kSubmit)) {
3612 0 : Info("StartAnalysis","\n##### SUBMIT MODE ##### Files required by your analysis are copied to your grid working \
3613 : \n space and job submitted.");
3614 0 : } else if (TestBit(AliAnalysisGrid::kMerge)) {
3615 0 : Info("StartAnalysis","\n##### MERGE MODE ##### The registered outputs of the analysis will be merged");
3616 0 : if (fMergeViaJDL) CheckInputData();
3617 0 : return kTRUE;
3618 : } else {
3619 0 : Info("StartAnalysis","\n##### FULL ANALYSIS MODE ##### Producing needed files and submitting your analysis job...");
3620 : }
3621 :
3622 0 : Print();
3623 0 : if (!Connect()) {
3624 0 : Error("StartAnalysis", "Cannot start grid analysis without grid connection");
3625 0 : return kFALSE;
3626 : }
3627 0 : if (IsCheckCopy() && gGrid) CheckFileCopy(gGrid->GetHomeDirectory());
3628 0 : if (!CheckInputData()) {
3629 0 : Error("StartAnalysis", "There was an error in preprocessing your requested input data");
3630 0 : return kFALSE;
3631 : }
3632 0 : if (!CreateDataset(fDataPattern)) {
3633 0 : TString serror;
3634 0 : if (!fRunNumbers.Length() && !fRunRange[0]) serror = Form("path to data directory: <%s>", fGridDataDir.Data());
3635 0 : if (fRunNumbers.Length()) serror = "run numbers";
3636 0 : if (fRunRange[0]) serror = Form("run range [%d, %d]", fRunRange[0], fRunRange[1]);
3637 0 : serror += Form("\n or data pattern <%s>", fDataPattern.Data());
3638 0 : Error("StartAnalysis", "No data to process. Please fix %s in your plugin configuration.", serror.Data());
3639 : return kFALSE;
3640 0 : }
3641 0 : WriteAnalysisFile();
3642 0 : WriteAnalysisMacro(nentries, firstEntry);
3643 0 : WriteExecutable();
3644 0 : WriteValidationScript();
3645 0 : if (fMergeViaJDL) {
3646 0 : WriteMergingMacro();
3647 0 : WriteMergeExecutable();
3648 0 : WriteValidationScript(kTRUE);
3649 0 : }
3650 0 : if (!CreateJDL()) return kFALSE;
3651 0 : if (TestBit(AliAnalysisGrid::kOffline)) return kFALSE;
3652 0 : if (testMode) {
3653 : // Locally testing the analysis
3654 0 : Info("StartAnalysis", "\n_______________________________________________________________________ \
3655 : \n Running analysis script in a daughter shell as on a worker node \
3656 : \n_______________________________________________________________________");
3657 0 : TObjArray *list = fOutputFiles.Tokenize(",");
3658 0 : TIter next(list);
3659 : TObjString *str;
3660 0 : TString outputFile;
3661 0 : while((str=(TObjString*)next())) {
3662 0 : outputFile = str->GetString();
3663 0 : Int_t index = outputFile.Index("@");
3664 0 : if (index > 0) outputFile.Remove(index);
3665 0 : if (!gSystem->AccessPathName(outputFile)) gSystem->Exec(Form("rm %s", outputFile.Data()));
3666 : }
3667 0 : delete list;
3668 0 : gSystem->Exec(Form("bash %s 2>stderr", fExecutable.Data()));
3669 0 : gSystem->Exec(Form("bash %s",fValidationScript.Data()));
3670 : // gSystem->Exec("cat stdout");
3671 : return kFALSE;
3672 0 : }
3673 : // Check if submitting is managed by LPM manager
3674 0 : if (fProductionMode) {
3675 : //TString prodfile = fJDLName;
3676 : //prodfile.ReplaceAll(".jdl", ".prod");
3677 : //WriteProductionFile(prodfile);
3678 0 : Info("StartAnalysis", "Job submitting is managed by LPM. Rerun in terminate mode after jobs finished.");
3679 0 : return kFALSE;
3680 : }
3681 : // Submit AliEn job(s)
3682 0 : gGrid->Cd(fGridOutputDir);
3683 : TGridResult *res;
3684 0 : TString jobID = "";
3685 0 : fGridJobIDs = "";
3686 0 : fGridStages = "";
3687 0 : if (!fRunNumbers.Length() && !fRunRange[0]) {
3688 : // Submit a given xml or a set of runs
3689 0 : res = gGrid->Command(Form("submit %s", fJDLName.Data()));
3690 0 : printf("*************************** %s\n",Form("submit %s", fJDLName.Data()));
3691 0 : if (res) {
3692 0 : const char *cjobId = res->GetKey(0,"jobId");
3693 0 : if (!cjobId) {
3694 0 : gGrid->Stdout();
3695 0 : gGrid->Stderr();
3696 0 : Error("StartAnalysis", "Your JDL %s could not be submitted", fJDLName.Data());
3697 0 : return kFALSE;
3698 : } else {
3699 0 : Info("StartAnalysis", "\n_______________________________________________________________________ \
3700 : \n##### Your JDL %s was successfully submitted. \nTHE JOB ID IS: %s \
3701 : \n_______________________________________________________________________",
3702 0 : fJDLName.Data(), cjobId);
3703 0 : jobID = cjobId;
3704 0 : if (jobID.Atoi()) {
3705 0 : if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3706 0 : fGridJobIDs.Append(jobID);
3707 0 : if (!fGridStages.IsNull()) fGridStages.Append(" ");
3708 0 : fGridStages.Append("full");
3709 : }
3710 : }
3711 0 : delete res;
3712 0 : } else {
3713 0 : Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3714 0 : return kFALSE;
3715 : }
3716 : } else {
3717 : // Submit for a range of enumeration of runs.
3718 0 : if (!Submit()) return kFALSE;
3719 0 : jobID = fGridJobIDs;
3720 : }
3721 :
3722 0 : if (fDropToShell) {
3723 0 : Info("StartAnalysis", "\n#### STARTING AN ALIEN SHELL FOR YOU. EXIT WHEN YOUR JOB %s HAS FINISHED. #### \
3724 : \n You may exit at any time and terminate the job later using the option <terminate> \
3725 0 : \n ##################################################################################", jobID.Data());
3726 0 : gSystem->Exec("aliensh");
3727 : } else {
3728 0 : Info("StartAnalysis", "\n#### SUBMITTED JOB %s TO ALIEN QUEUE #### \
3729 : \n Remember to terminate the job later using the option <terminate> \
3730 0 : \n ##################################################################################", jobID.Data());
3731 : }
3732 0 : return kTRUE;
3733 0 : }
3734 :
3735 : //______________________________________________________________________________
3736 : const char *AliAnalysisAlien::GetListOfFiles(const char *type)
3737 : {
3738 : // Get a comma-separated list of output files of the requested type.
3739 : // Type can be (case unsensitive):
3740 : // aod - list of aod files (std, extensions and filters)
3741 : // out - list of output files connected to containers (but not aod's or extras)
3742 : // ext - list of extra files registered to the manager
3743 : // ter - list of files produced in terminate
3744 0 : static TString files;
3745 0 : files = "";
3746 0 : TString stype = type;
3747 0 : stype.ToLower();
3748 0 : TString aodfiles, extra;
3749 0 : AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
3750 0 : if (!mgr) {
3751 0 : ::Error("GetListOfFiles", "Cannot call this without analysis manager");
3752 0 : return files.Data();
3753 : }
3754 0 : if (mgr->GetOutputEventHandler()) {
3755 0 : aodfiles = "";
3756 0 : if (mgr->GetOutputEventHandler()->GetFillAOD())
3757 0 : aodfiles = mgr->GetOutputEventHandler()->GetOutputFileName();
3758 0 : TString extraaod = mgr->GetOutputEventHandler()->GetExtraOutputs(kTRUE);
3759 0 : if (!extraaod.IsNull() && mgr->GetOutputEventHandler()->GetFillExtension()) {
3760 0 : if (!aodfiles.IsNull()) aodfiles += ",";
3761 0 : aodfiles += extraaod;
3762 : }
3763 0 : }
3764 0 : if (stype.Contains("aod")) {
3765 0 : files = aodfiles;
3766 0 : if (stype == "aod") return files.Data();
3767 : }
3768 : // Add output files that are not in the list of AOD files
3769 0 : TString outputfiles = "";
3770 0 : TIter next(mgr->GetOutputs());
3771 : AliAnalysisDataContainer *output;
3772 : const char *filename = 0;
3773 0 : while ((output=(AliAnalysisDataContainer*)next())) {
3774 0 : filename = output->GetFileName();
3775 0 : if (!(strcmp(filename, "default"))) continue;
3776 0 : if (outputfiles.Contains(filename)) continue;
3777 0 : if (aodfiles.Contains(filename)) continue;
3778 0 : if (!outputfiles.IsNull() && strlen(filename)) outputfiles += ",";
3779 0 : outputfiles += filename;
3780 : }
3781 0 : if (stype.Contains("out")) {
3782 0 : if (!files.IsNull() && !outputfiles.IsNull()) files += ",";
3783 0 : files += outputfiles;
3784 0 : if (stype == "out") return files.Data();
3785 : }
3786 : // Add extra files registered to the analysis manager
3787 0 : TString sextra;
3788 0 : extra = mgr->GetExtraFiles();
3789 0 : if (!extra.IsNull()) {
3790 0 : extra.Strip();
3791 0 : extra.ReplaceAll(" ", ",");
3792 0 : TObjArray *fextra = extra.Tokenize(",");
3793 0 : TIter nextx(fextra);
3794 : TObject *obj;
3795 0 : while ((obj=nextx())) {
3796 0 : if (aodfiles.Contains(obj->GetName())) continue;
3797 0 : if (outputfiles.Contains(obj->GetName())) continue;
3798 0 : if (sextra.Contains(obj->GetName())) continue;
3799 0 : if (!sextra.IsNull()) sextra += ",";
3800 0 : sextra += obj->GetName();
3801 : }
3802 0 : delete fextra;
3803 0 : if (stype.Contains("ext")) {
3804 0 : if (!files.IsNull() && !sextra.IsNull()) files += ",";
3805 0 : files += sextra;
3806 : }
3807 0 : }
3808 0 : if (stype == "ext") return files.Data();
3809 0 : TString termfiles;
3810 0 : if (!fTerminateFiles.IsNull()) {
3811 0 : fTerminateFiles.Strip();
3812 0 : fTerminateFiles.ReplaceAll(" ",",");
3813 0 : TObjArray *fextra = fTerminateFiles.Tokenize(",");
3814 0 : TIter nextx(fextra);
3815 : TObject *obj;
3816 0 : while ((obj=nextx())) {
3817 0 : if (aodfiles.Contains(obj->GetName())) continue;
3818 0 : if (outputfiles.Contains(obj->GetName())) continue;
3819 0 : if (termfiles.Contains(obj->GetName())) continue;
3820 0 : if (sextra.Contains(obj->GetName())) continue;
3821 0 : if (!termfiles.IsNull()) termfiles += ",";
3822 0 : termfiles += obj->GetName();
3823 : }
3824 0 : delete fextra;
3825 0 : }
3826 0 : if (stype.Contains("ter")) {
3827 0 : if (!files.IsNull() && !termfiles.IsNull()) {
3828 0 : files += ",";
3829 0 : files += termfiles;
3830 : }
3831 : }
3832 0 : return files.Data();
3833 0 : }
3834 :
3835 : //______________________________________________________________________________
3836 : Bool_t AliAnalysisAlien::Submit()
3837 : {
3838 : // Submit all master jobs.
3839 0 : Int_t nmasterjobs = fInputFiles->GetEntries();
3840 0 : Long_t tshoot = gSystem->Now();
3841 0 : if (!fNsubmitted && !SubmitNext()) return kFALSE;
3842 0 : while (fNsubmitted < nmasterjobs) {
3843 0 : Long_t now = gSystem->Now();
3844 0 : if ((now-tshoot)>30000) {
3845 : tshoot = now;
3846 0 : if (!SubmitNext()) return kFALSE;
3847 : }
3848 0 : }
3849 0 : return kTRUE;
3850 0 : }
3851 :
3852 : //______________________________________________________________________________
3853 : Bool_t AliAnalysisAlien::SubmitMerging()
3854 : {
3855 : // Submit all merging jobs.
3856 0 : if (!fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
3857 0 : gGrid->Cd(fGridOutputDir);
3858 0 : TString mergeJDLName = fExecutable;
3859 0 : mergeJDLName.ReplaceAll(".sh", "_merge.jdl");
3860 0 : if (!fInputFiles) {
3861 0 : Error("SubmitMerging", "You have to use explicit run numbers or run range to merge via JDL!");
3862 0 : return kFALSE;
3863 : }
3864 0 : Int_t ntosubmit = fInputFiles->GetEntries();
3865 0 : for (Int_t i=0; i<ntosubmit; i++) {
3866 0 : TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
3867 0 : runOutDir.ReplaceAll(".xml", "");
3868 0 : if (fOutputToRunNo) {
3869 : // The output directory is the run number
3870 0 : printf("### Submitting merging job for run <%s>\n", runOutDir.Data());
3871 0 : runOutDir = Form("%s/%s", fGridOutputDir.Data(), runOutDir.Data());
3872 : } else {
3873 0 : if (!fRunNumbers.Length() && !fRunRange[0]) {
3874 : // The output directory is the grid outdir
3875 0 : printf("### Submitting merging job for the full output directory %s.\n", fGridOutputDir.Data());
3876 0 : runOutDir = fGridOutputDir;
3877 : } else {
3878 : // The output directory is the master number in 3 digits format
3879 0 : printf("### Submitting merging job for master <%03d>\n", i);
3880 0 : runOutDir = Form("%s/%03d",fGridOutputDir.Data(), i);
3881 : }
3882 : }
3883 : // Check now the number of merging stages.
3884 0 : TObjArray *list = fOutputFiles.Tokenize(",");
3885 0 : TIter next(list);
3886 : TObjString *str;
3887 0 : TString outputFile;
3888 0 : while((str=(TObjString*)next())) {
3889 0 : outputFile = str->GetString();
3890 0 : Int_t index = outputFile.Index("@");
3891 0 : if (index > 0) outputFile.Remove(index);
3892 0 : if (!fMergeExcludes.Contains(outputFile) &&
3893 0 : !fRegisterExcludes.Contains(outputFile)) break;
3894 0 : }
3895 0 : delete list;
3896 0 : Bool_t done = CheckMergedFiles(outputFile, runOutDir, fMaxMergeFiles, mergeJDLName);
3897 0 : if (!done && (i==ntosubmit-1)) return kFALSE;
3898 0 : if (!fRunNumbers.Length() && !fRunRange[0]) break;
3899 0 : }
3900 0 : if (!ntosubmit) return kTRUE;
3901 0 : if (fDropToShell) {
3902 0 : Info("StartAnalysis", "\n #### STARTING AN ALIEN SHELL FOR YOU. You can exit any time or inspect your jobs in a different shell.##########\
3903 : \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL')\
3904 : \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3905 : \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3906 : \n ################################################################################################################");
3907 0 : gSystem->Exec("aliensh");
3908 : } else {
3909 0 : Info("StartAnalysis", "\n #### STARTED MERGING JOBS FOR YOU #### \
3910 : \n Make sure your jobs are in a final state (you can resubmit failed ones via 'masterjob <id> resubmit ERROR_ALL') \
3911 : \n Rerun in 'terminate' mode to submit all merging stages, each AFTER the previous one completed. The final merged \
3912 : \n output will be written to your alien output directory, while separate stages in <Stage_n>. \
3913 : \n ################################################################################################################");
3914 : }
3915 0 : return kTRUE;
3916 0 : }
3917 :
3918 : //______________________________________________________________________________
3919 : Bool_t AliAnalysisAlien::SubmitNext()
3920 : {
3921 : // Submit next bunch of master jobs if the queue is free. The first master job is
3922 : // submitted right away, while the next will not be unless the previous was split.
3923 : // The plugin will not submit new master jobs if there are more that 500 jobs in
3924 : // waiting phase.
3925 : static Bool_t iscalled = kFALSE;
3926 : static Int_t firstmaster = 0;
3927 : static Int_t lastmaster = 0;
3928 : static Int_t npermaster = 0;
3929 0 : if (iscalled) return kTRUE;
3930 0 : iscalled = kTRUE;
3931 0 : Int_t nrunning=0, nwaiting=0, nerror=0, ndone=0;
3932 : Int_t ntosubmit = 0;
3933 : TGridResult *res;
3934 0 : TString jobID = "";
3935 0 : Int_t nmasterjobs = fInputFiles->GetEntries();
3936 0 : if (!fNsubmitted) {
3937 : ntosubmit = 1;
3938 0 : if (!IsUseSubmitPolicy()) {
3939 0 : if (nmasterjobs>5)
3940 0 : Info("SubmitNext","### Warning submit policy not used ! Submitting too many jobs at a time may be prohibitted. \
3941 : \n### You can use SetUseSubmitPolicy() to enable if you have problems.");
3942 : ntosubmit = nmasterjobs;
3943 0 : }
3944 : } else {
3945 0 : TString status = GetJobStatus(firstmaster, lastmaster, nrunning, nwaiting, nerror, ndone);
3946 0 : printf("=== master %d: %s\n", lastmaster, status.Data());
3947 : // If last master not split, just return
3948 0 : if (status != "SPLIT") {iscalled = kFALSE; return kTRUE;}
3949 : // No more than 100 waiting jobs
3950 0 : if (nwaiting>500) {iscalled = kFALSE; return kTRUE;}
3951 0 : npermaster = (nrunning+nwaiting+nerror+ndone)/fNsubmitted;
3952 0 : if (npermaster) ntosubmit = (500-nwaiting)/npermaster;
3953 0 : if (!ntosubmit) ntosubmit = 1;
3954 0 : printf("=== WAITING(%d) RUNNING(%d) DONE(%d) OTHER(%d) NperMaster=%d => to submit %d jobs\n",
3955 0 : nwaiting, nrunning, ndone, nerror, npermaster, ntosubmit);
3956 0 : }
3957 0 : for (Int_t i=0; i<ntosubmit; i++) {
3958 : // Submit for a range of enumeration of runs.
3959 0 : if (fNsubmitted>=nmasterjobs) {iscalled = kFALSE; return kTRUE;}
3960 0 : TString query;
3961 0 : TString runOutDir = gSystem->BaseName(fInputFiles->At(fNsubmitted)->GetName());
3962 0 : runOutDir.ReplaceAll(".xml", "");
3963 0 : if (fOutputToRunNo)
3964 0 : query = Form("submit %s %s %s", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), runOutDir.Data());
3965 : else
3966 0 : query = Form("submit %s %s %03d", fJDLName.Data(), fInputFiles->At(fNsubmitted)->GetName(), fNsubmitted);
3967 0 : printf("********* %s\n",query.Data());
3968 0 : res = gGrid->Command(query);
3969 0 : if (res) {
3970 0 : TString cjobId1 = res->GetKey(0,"jobId");
3971 0 : if (!cjobId1.Length()) {
3972 0 : iscalled = kFALSE;
3973 0 : gGrid->Stdout();
3974 0 : gGrid->Stderr();
3975 0 : Error("StartAnalysis", "Your JDL %s could not be submitted. The message was:", fJDLName.Data());
3976 0 : return kFALSE;
3977 : } else {
3978 0 : Info("StartAnalysis", "\n_______________________________________________________________________ \
3979 : \n##### Your JDL %s submitted (%d to go). \nTHE JOB ID IS: %s \
3980 : \n_______________________________________________________________________",
3981 0 : fJDLName.Data(), nmasterjobs-fNsubmitted-1, cjobId1.Data());
3982 0 : if (!fGridJobIDs.IsNull()) fGridJobIDs.Append(" ");
3983 0 : fGridJobIDs.Append(cjobId1);
3984 0 : if (!fGridStages.IsNull()) fGridStages.Append(" ");
3985 0 : fGridStages.Append("full");
3986 0 : jobID += cjobId1;
3987 0 : jobID += " ";
3988 0 : lastmaster = cjobId1.Atoi();
3989 0 : if (!firstmaster) firstmaster = lastmaster;
3990 0 : fNsubmitted++;
3991 : }
3992 0 : delete res;
3993 0 : } else {
3994 0 : Error("StartAnalysis", "No grid result after submission !!! Bailing out...");
3995 0 : return kFALSE;
3996 : }
3997 0 : }
3998 0 : iscalled = kFALSE;
3999 0 : return kTRUE;
4000 0 : }
4001 :
4002 : //______________________________________________________________________________
4003 : void AliAnalysisAlien::WriteAnalysisFile()
4004 : {
4005 : // Write current analysis manager into the file <analysisFile>
4006 0 : TString analysisFile = fExecutable;
4007 0 : analysisFile.ReplaceAll(".sh", ".root");
4008 0 : if (!TestBit(AliAnalysisGrid::kSubmit)) {
4009 0 : AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
4010 0 : if (!mgr || !mgr->IsInitialized()) {
4011 0 : Error("WriteAnalysisFile", "You need an initialized analysis manager for this");
4012 0 : return;
4013 : }
4014 : // Check analysis type
4015 : TObject *handler;
4016 0 : if (mgr->GetMCtruthEventHandler()) TObject::SetBit(AliAnalysisGrid::kUseMC);
4017 0 : handler = (TObject*)mgr->GetInputEventHandler();
4018 0 : if (handler) {
4019 0 : if (handler->InheritsFrom("AliMultiInputEventHandler")) {
4020 0 : AliMultiInputEventHandler *multiIH = (AliMultiInputEventHandler*)handler;
4021 0 : if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
4022 0 : if (multiIH->GetFirstInputEventHandler()->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
4023 0 : } else {
4024 0 : if (handler->InheritsFrom("AliESDInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseESD);
4025 0 : if (handler->InheritsFrom("AliAODInputHandler")) TObject::SetBit(AliAnalysisGrid::kUseAOD);
4026 : }
4027 : }
4028 0 : TDirectory *cdir = gDirectory;
4029 0 : TFile *file = TFile::Open(analysisFile, "RECREATE");
4030 0 : if (file) {
4031 : // Skip task Terminate calls for the grid job (but not in test mode, where we want to check also the terminate mode
4032 0 : if (!TestBit(AliAnalysisGrid::kTest)) mgr->SetSkipTerminate(kTRUE);
4033 : // Unless merging makes no sense
4034 0 : if (IsSingleOutput()) mgr->SetSkipTerminate(kFALSE);
4035 0 : mgr->Write();
4036 0 : delete file;
4037 : // Enable termination for local jobs
4038 0 : mgr->SetSkipTerminate(kFALSE);
4039 : }
4040 0 : if (cdir) cdir->cd();
4041 0 : Info("WriteAnalysisFile", "\n##### Analysis manager: %s wrote to file <%s>\n", mgr->GetName(),analysisFile.Data());
4042 0 : }
4043 : Bool_t copy = kTRUE;
4044 0 : if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
4045 0 : if (copy) {
4046 0 : CdWork();
4047 0 : TString workdir = gGrid->GetHomeDirectory();
4048 0 : workdir += fGridWorkingDir;
4049 0 : Info("WriteAnalysisFile", "\n##### Copying file <%s> containing your initialized analysis manager to your alien workspace", analysisFile.Data());
4050 0 : if (FileExists(analysisFile)) gGrid->Rm(analysisFile);
4051 0 : if (!copyLocal2Alien("WriteAnalysisFile",analysisFile.Data(),
4052 0 : Form("%s/%s", workdir.Data(),analysisFile.Data()))) Fatal("","Terminating");
4053 0 : }
4054 0 : }
4055 :
4056 : //______________________________________________________________________________
4057 : void AliAnalysisAlien::WriteAnalysisMacro(Long64_t nentries, Long64_t firstentry)
4058 : {
4059 : // Write the analysis macro that will steer the analysis in grid mode.
4060 0 : if (!TestBit(AliAnalysisGrid::kSubmit)) {
4061 0 : ofstream out;
4062 0 : out.open(fAnalysisMacro.Data(), ios::out);
4063 0 : if (!out.good()) {
4064 0 : Error("WriteAnalysisMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4065 0 : return;
4066 : }
4067 : Bool_t hasSTEERBase = kFALSE;
4068 : Bool_t hasESD = kFALSE;
4069 : Bool_t hasAOD = kFALSE;
4070 : Bool_t hasANALYSIS = kFALSE;
4071 : Bool_t hasOADB = kFALSE;
4072 : Bool_t hasANALYSISalice = kFALSE;
4073 : Bool_t hasCORRFW = kFALSE;
4074 0 : TString func = fAnalysisMacro;
4075 0 : TString type = "ESD";
4076 0 : TString comment = "// Analysis using ";
4077 0 : if (fMCLoop) {
4078 0 : type = "MCGEN";
4079 0 : comment += "MCGEN";
4080 : } else {
4081 0 : if (IsUseMCchain()) {
4082 0 : type = "MC";
4083 0 : comment += "MC";
4084 : } else {
4085 0 : if (TObject::TestBit(AliAnalysisGrid::kUseESD)) comment += "ESD";
4086 0 : if (TObject::TestBit(AliAnalysisGrid::kUseAOD)) {
4087 0 : type = "AOD";
4088 0 : comment += "AOD";
4089 : }
4090 : }
4091 : }
4092 0 : if (type!="AOD" && fFriendChainName!="") {
4093 0 : Error("WriteAnalysisMacro", "Friend chain can be attached only to AOD");
4094 0 : return;
4095 : }
4096 0 : if (TObject::TestBit(AliAnalysisGrid::kUseMC)) comment += "/MC";
4097 0 : else comment += " data";
4098 0 : out << "TChain* CreateChain(const char *xmlfile, const char *type=\"ESD\");" << endl << endl;
4099 0 : out << "const char *anatype = \"" << type.Data() << "\";" << endl << endl;
4100 0 : func.ReplaceAll(".C", "");
4101 0 : out << "void " << func.Data() << "()" << endl;
4102 0 : out << "{" << endl;
4103 0 : out << comment.Data() << endl;
4104 0 : out << "// Automatically generated analysis steering macro executed in grid subjobs" << endl << endl;
4105 0 : out << " TStopwatch timer;" << endl;
4106 0 : out << " timer.Start();" << endl << endl;
4107 : // Change temp directory to current one
4108 0 : if (!IsLocalTest()) {
4109 0 : out << "// connect to AliEn and make the chain" << endl;
4110 0 : out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
4111 : }
4112 0 : out << "// Set temporary merging directory to current one" << endl;
4113 0 : out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
4114 0 : out << "// Set temporary compilation directory to current one" << endl;
4115 0 : out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
4116 : // Reset existing include path
4117 0 : out << "// Reset existing include path and add current directory first in the search" << endl;
4118 0 : out << " gSystem->SetIncludePath(\"-I.\");" << endl;
4119 0 : if (!fExecutableCommand.Contains("aliroot")) {
4120 0 : out << "// load base root libraries" << endl;
4121 0 : out << " gSystem->Load(\"libTree\");" << endl;
4122 0 : out << " gSystem->Load(\"libGeom\");" << endl;
4123 0 : out << " gSystem->Load(\"libVMC\");" << endl;
4124 0 : out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4125 0 : out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4126 : }
4127 0 : if (fAdditionalRootLibs.Length()) {
4128 : // in principle libtree /lib geom libvmc etc. can go into this list, too
4129 0 : out << "// Add aditional libraries" << endl;
4130 0 : TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4131 0 : TIter next(list);
4132 : TObjString *str;
4133 0 : TString buf;
4134 0 : while((str=(TObjString*)next())) {
4135 0 : buf = str->GetString();
4136 0 : if (buf.Contains(".so") || buf.Contains(".dylib")) {
4137 0 : buf.ReplaceAll(".so", "");
4138 0 : buf.ReplaceAll(".dylib", "");
4139 0 : out << " gSystem->Load(\"" << buf.Data() << "\");" << endl;
4140 : }
4141 : }
4142 0 : if (list) delete list;
4143 0 : }
4144 0 : out << "// Load analysis framework libraries" << endl;
4145 0 : TString setupPar = "AliAnalysisAlien::SetupPar";
4146 0 : if (!fPackages) {
4147 0 : if (!fExecutableCommand.Contains("aliroot")) {
4148 0 : out << " gSystem->Load(\"libSTEERBase\");" << endl;
4149 0 : out << " gSystem->Load(\"libESD\");" << endl;
4150 0 : out << " gSystem->Load(\"libAOD\");" << endl;
4151 : }
4152 0 : out << " gSystem->Load(\"libANALYSIS\");" << endl;
4153 0 : out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4154 0 : out << " gSystem->Load(\"libOADB\");" << endl;
4155 0 : out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4156 : } else {
4157 0 : TIter next(fPackages);
4158 : TObject *obj;
4159 0 : TString pkgname;
4160 0 : while ((obj=next())) {
4161 0 : pkgname = obj->GetName();
4162 0 : if (pkgname == "STEERBase" ||
4163 0 : pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4164 0 : if (pkgname == "ESD" ||
4165 0 : pkgname == "ESD.par") hasESD = kTRUE;
4166 0 : if (pkgname == "AOD" ||
4167 0 : pkgname == "AOD.par") hasAOD = kTRUE;
4168 0 : if (pkgname == "ANALYSIS" ||
4169 0 : pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
4170 0 : if (pkgname == "OADB" ||
4171 0 : pkgname == "OADB.par") hasOADB = kTRUE;
4172 0 : if (pkgname == "ANALYSISalice" ||
4173 0 : pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4174 0 : if (pkgname == "CORRFW" ||
4175 0 : pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4176 : }
4177 0 : if (hasANALYSISalice) setupPar = "SetupPar";
4178 0 : if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
4179 0 : else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
4180 0 : if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
4181 0 : else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
4182 0 : if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
4183 0 : else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
4184 0 : if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
4185 0 : else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
4186 0 : if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4187 0 : else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
4188 0 : if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4189 0 : else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
4190 0 : if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4191 0 : else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
4192 0 : out << "// Compile other par packages" << endl;
4193 0 : next.Reset();
4194 0 : while ((obj=next())) {
4195 0 : pkgname = obj->GetName();
4196 0 : if (pkgname == "STEERBase" ||
4197 0 : pkgname == "STEERBase.par" ||
4198 0 : pkgname == "ESD" ||
4199 0 : pkgname == "ESD.par" ||
4200 0 : pkgname == "AOD" ||
4201 0 : pkgname == "AOD.par" ||
4202 0 : pkgname == "ANALYSIS" ||
4203 0 : pkgname == "ANALYSIS.par" ||
4204 0 : pkgname == "OADB" ||
4205 0 : pkgname == "OADB.par" ||
4206 0 : pkgname == "ANALYSISalice" ||
4207 0 : pkgname == "ANALYSISalice.par" ||
4208 0 : pkgname == "CORRFW" ||
4209 0 : pkgname == "CORRFW.par") continue;
4210 0 : out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
4211 : }
4212 0 : }
4213 0 : out << "// include path" << endl;
4214 : // Get the include path from the interpreter and remove entries pointing to AliRoot
4215 0 : out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4216 0 : out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4217 0 : out << " TIter nextpath(listpaths);" << endl;
4218 0 : out << " TObjString *pname;" << endl;
4219 0 : out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4220 0 : out << " TString current = pname->GetName();" << endl;
4221 0 : out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4222 0 : out << " gSystem->AddIncludePath(current);" << endl;
4223 0 : out << " }" << endl;
4224 0 : out << " if (listpaths) delete listpaths;" << endl;
4225 0 : if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4226 0 : out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4227 0 : out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
4228 0 : if (fMCLoop && !fGeneratorLibs.IsNull()) {
4229 0 : out << "// MC generator libraries" << endl;
4230 0 : TObjArray *list = fGeneratorLibs.Tokenize(" ");
4231 0 : TIter next(list);
4232 : TObjString *str;
4233 0 : while((str=(TObjString*)next())) {
4234 0 : out << " gSystem->Load(\"" << str->GetName() << "\");" << endl;
4235 : }
4236 0 : delete list;
4237 0 : }
4238 0 : if (fAdditionalLibs.Length()) {
4239 0 : out << "// Add aditional AliRoot libraries" << endl;
4240 0 : TString additionalLibs = fAdditionalLibs;
4241 0 : additionalLibs.Strip();
4242 0 : if (additionalLibs.Length() && fFriendLibs.Length())
4243 0 : additionalLibs += " ";
4244 0 : additionalLibs += fFriendLibs;
4245 0 : TObjArray *list = additionalLibs.Tokenize(" ");
4246 0 : TIter next(list);
4247 : TObjString *str;
4248 0 : TString buf;
4249 0 : while((str=(TObjString*)next())) {
4250 0 : buf = str->GetString();
4251 0 : if (buf.Contains(".so") || buf.Contains(".dylib")) {
4252 0 : buf.ReplaceAll(".so", "");
4253 0 : buf.ReplaceAll(".dylib", "");
4254 0 : out << " gSystem->Load(\"" << buf.Data() << "\");" << endl;
4255 : }
4256 0 : if (buf.Contains(".par"))
4257 0 : out << " if (!" << setupPar << "(\"" << buf.Data() << "\")) return;" << endl;
4258 : }
4259 0 : delete list;
4260 0 : }
4261 0 : out << endl;
4262 0 : out << "// analysis source to be compiled at runtime (if any)" << endl;
4263 0 : if (fAnalysisSource.Length()) {
4264 0 : TObjArray *list = fAnalysisSource.Tokenize(" ");
4265 0 : TIter next(list);
4266 : TObjString *str;
4267 0 : while((str=(TObjString*)next())) {
4268 0 : out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4269 : }
4270 0 : if (list) delete list;
4271 0 : }
4272 0 : out << endl;
4273 : // out << " printf(\"Currently load libraries:\\n\");" << endl;
4274 : // out << " printf(\"%s\\n\", gSystem->GetLibraries());" << endl;
4275 0 : if (fFastReadOption) {
4276 0 : Warning("WriteAnalysisMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid jobs. This may skip some files that could be accessed !!! \
4277 : \n+++ NOTE: To disable this option, use: plugin->SetFastReadOption(kFALSE)");
4278 0 : out << "// fast xrootd reading enabled" << endl;
4279 0 : out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
4280 0 : out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4281 0 : out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
4282 0 : out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
4283 0 : out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
4284 0 : out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
4285 : }
4286 0 : out << "// read the analysis manager from file" << endl;
4287 0 : TString analysisFile = fExecutable;
4288 0 : analysisFile.ReplaceAll(".sh", ".root");
4289 0 : out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4290 0 : << analysisFile << "\");" << endl;
4291 0 : out << " if (!mgr) return;" << endl;
4292 0 : if (IsLocalTest()) {
4293 0 : out << " AliAnalysisAlien *plugin = new AliAnalysisAlien();" << endl;
4294 0 : out << " plugin->SetRunMode(\"test\");" << endl;
4295 0 : if (fFileForTestMode.IsNull())
4296 0 : out << " plugin->SetFileForTestMode(\"data.txt\");" << endl;
4297 : else
4298 0 : out << " plugin->SetFileForTestMode(\"" << fFileForTestMode << "\");" << endl;
4299 0 : out << " plugin->SetNtestFiles(" << fNtestFiles << ");" << endl;
4300 0 : if (!fFriendChainName.IsNull())
4301 0 : out << " plugin->SetFriendChainName(\"" << fFriendChainName << "\",\"" << fFriendLibs << "\");" << endl;
4302 0 : if (IsUseMCchain())
4303 0 : out << " plugin->SetUseMCchain();" << endl;
4304 0 : if (fMCLoop)
4305 0 : out << " plugin->SetMCLoop(kTRUE);" << endl;
4306 0 : out << " mgr->SetGridHandler(plugin);" << endl;
4307 0 : if (AliAnalysisManager::GetAnalysisManager()) {
4308 0 : out << " mgr->SetDebugLevel(" << AliAnalysisManager::GetAnalysisManager()->GetDebugLevel() << ");" << endl;
4309 0 : out << " mgr->SetNSysInfo(" << AliAnalysisManager::GetAnalysisManager()->GetNsysInfo() << ");" << endl;
4310 : } else {
4311 0 : out << " mgr->SetDebugLevel(10);" << endl;
4312 0 : out << " mgr->SetNSysInfo(100);" << endl;
4313 : }
4314 : }
4315 0 : out << " mgr->PrintStatus();" << endl;
4316 0 : if (AliAnalysisManager::GetAnalysisManager()) {
4317 0 : if (AliAnalysisManager::GetAnalysisManager()->GetDebugLevel()>3) {
4318 0 : out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4319 : } else {
4320 0 : if (TestBit(AliAnalysisGrid::kTest))
4321 0 : out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4322 : else
4323 0 : out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
4324 : }
4325 : }
4326 0 : if (!IsLocalTest()) {
4327 0 : if (fMCLoop) {
4328 0 : out << " mgr->SetCacheSize(0);" << endl;
4329 0 : out << " mgr->EventLoop(" << fNMCevents << ");" << endl;
4330 : } else {
4331 0 : out << " TChain *chain = CreateChain(\"wn.xml\", anatype);" << endl << endl;
4332 0 : out << " mgr->StartAnalysis(\"localfile\", chain, "
4333 0 : << nentries << ", " << firstentry << ");" << endl;
4334 : }
4335 : } else {
4336 0 : if (fMCLoop) {
4337 0 : out << " mgr->SetCacheSize(0);" << endl;
4338 0 : out << " mgr->EventLoop(" << fNMCevents << ");" << endl;
4339 : } else {
4340 0 : out << " mgr->StartAnalysis(\"localfile\", "
4341 0 : << nentries << ", " << firstentry << ");" << endl;
4342 : }
4343 : }
4344 0 : out << " timer.Stop();" << endl;
4345 0 : out << " timer.Print();" << endl;
4346 0 : out << "}" << endl << endl;
4347 0 : if (!IsLocalTest() && !fMCLoop) {
4348 0 : out <<"//________________________________________________________________________________" << endl;
4349 0 : out << "TChain* CreateChain(const char *xmlfile, const char *type)" << endl;
4350 0 : out << "{" << endl;
4351 0 : out << "// Create a chain using url's from xml file" << endl;
4352 0 : out << " TString filename;" << endl;
4353 0 : out << " Int_t run = 0;" << endl;
4354 0 : if (IsUseMCchain()) {
4355 0 : out << " TString treename = \"TE\";" << endl;
4356 : } else {
4357 0 : if (!fTreeName.IsNull()) {
4358 0 : out << " TString treename = \"" << fTreeName << "\";" << endl;
4359 : } else {
4360 0 : out << " TString treename = type;" << endl;
4361 0 : out << " treename.ToLower();" << endl;
4362 0 : out << " treename += \"Tree\";" << endl;
4363 : }
4364 : }
4365 0 : out << " printf(\"***************************************\\n\");" << endl;
4366 0 : out << " printf(\" Getting chain of trees %s\\n\", treename.Data());" << endl;
4367 0 : out << " printf(\"***************************************\\n\");" << endl;
4368 0 : out << " TAlienCollection *coll = dynamic_cast<TAlienCollection *>(TAlienCollection::Open(xmlfile));" << endl;
4369 0 : out << " if (!coll) {" << endl;
4370 0 : out << " ::Error(\"CreateChain\", \"Cannot create an AliEn collection from %s\", xmlfile);" << endl;
4371 0 : out << " return NULL;" << endl;
4372 0 : out << " }" << endl;
4373 0 : out << " AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();" << endl;
4374 0 : out << " TChain *chain = new TChain(treename);" << endl;
4375 0 : if(!fFriendChainName.IsNull()) {
4376 0 : out << " TList *friends = new TList();" << endl;
4377 0 : out << " TIter nextfriend(friends);" << endl;
4378 0 : out << " TChain *cfriend = 0;" << endl;
4379 0 : TObjArray *list = fFriendChainName.Tokenize(" ");
4380 0 : TIter next(list);
4381 : TObjString *str;
4382 0 : while((str=(TObjString*)next())) {
4383 0 : out << " cfriend = new TChain(treename, \"" << str->GetName() << "\");" << endl;
4384 0 : out << " friends->Add(cfriend);" << endl;
4385 0 : out << " chain->AddFriend(cfriend);" << endl;
4386 : }
4387 0 : delete list;
4388 : // out << " TChain *chainFriend = new TChain(treename);" << endl;
4389 0 : }
4390 0 : out << " coll->Reset();" << endl;
4391 0 : out << " while (coll->Next()) {" << endl;
4392 0 : out << " filename = coll->GetTURL("");" << endl;
4393 0 : out << " if (mgr) {" << endl;
4394 0 : out << " Int_t nrun = AliAnalysisManager::GetRunFromAlienPath(filename);" << endl;
4395 0 : out << " if (nrun && nrun != run) {" << endl;
4396 0 : out << " printf(\"### Run number detected from chain: %d\\n\", nrun);" << endl;
4397 0 : out << " mgr->SetRunFromPath(nrun);" << endl;
4398 0 : out << " run = nrun;" << endl;
4399 0 : out << " }" << endl;
4400 0 : out << " }" << endl;
4401 0 : out << " chain->Add(filename);" << endl;
4402 0 : if(!fFriendChainName.IsNull()) {
4403 0 : out << " TString bpath=coll->GetTURL(\"\");" << endl;
4404 0 : out << " if (bpath.Index(\"#\") > -1) bpath.Remove(bpath.Index(\"#\"));" << endl;
4405 0 : out << " bpath = gSystem->DirName(bpath);" << endl;
4406 0 : out << " bpath += \"/\";" << endl;
4407 0 : out << " TString fileFriend;" << endl;
4408 0 : out << " nextfriend.Reset();" << endl;
4409 0 : out << " while ((cfriend=(TChain*)nextfriend())) {" << endl;
4410 0 : out << " fileFriend = bpath;" << endl;
4411 0 : out << " fileFriend += cfriend->GetTitle();" << endl;
4412 0 : out << " TFile *file = TFile::Open(fileFriend);" << endl;
4413 0 : out << " if (file) {" << endl;
4414 0 : out << " file->Close();" << endl;
4415 0 : out << " cfriend->Add(fileFriend.Data());" << endl;
4416 0 : out << " } else {" << endl;
4417 0 : out << " ::Fatal(\"CreateChain\", \"Cannot open friend file: %s\", fileFriend.Data());" << endl;
4418 0 : out << " return 0;" << endl;
4419 0 : out << " }" << endl;
4420 0 : out << " }" << endl;
4421 : }
4422 0 : out << " }" << endl;
4423 0 : out << " if (!chain->GetNtrees()) {" << endl;
4424 0 : out << " ::Error(\"CreateChain\", \"No tree found from collection %s\", xmlfile);" << endl;
4425 0 : out << " return NULL;" << endl;
4426 0 : out << " }" << endl;
4427 0 : out << " return chain;" << endl;
4428 0 : out << "}" << endl << endl;
4429 : }
4430 0 : if (hasANALYSISalice) {
4431 0 : out <<"//________________________________________________________________________________" << endl;
4432 0 : out << "Bool_t SetupPar(const char *package) {" << endl;
4433 0 : out << "// Compile the package and set it up." << endl;
4434 0 : out << " TString pkgdir = package;" << endl;
4435 0 : out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
4436 0 : out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
4437 0 : out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4438 0 : out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4439 0 : out << " // Check for BUILD.sh and execute" << endl;
4440 0 : out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4441 0 : out << " printf(\"*******************************\\n\");" << endl;
4442 0 : out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4443 0 : out << " printf(\"*******************************\\n\");" << endl;
4444 0 : out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4445 0 : out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4446 0 : out << " gSystem->ChangeDirectory(cdir);" << endl;
4447 0 : out << " return kFALSE;" << endl;
4448 0 : out << " }" << endl;
4449 0 : out << " } else {" << endl;
4450 0 : out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4451 0 : out << " gSystem->ChangeDirectory(cdir);" << endl;
4452 0 : out << " return kFALSE;" << endl;
4453 0 : out << " }" << endl;
4454 0 : out << " // Check for SETUP.C and execute" << endl;
4455 0 : out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4456 0 : out << " printf(\"*******************************\\n\");" << endl;
4457 0 : out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4458 0 : out << " printf(\"*******************************\\n\");" << endl;
4459 0 : out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4460 0 : out << " } else {" << endl;
4461 0 : out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4462 0 : out << " gSystem->ChangeDirectory(cdir);" << endl;
4463 0 : out << " return kFALSE;" << endl;
4464 0 : out << " }" << endl;
4465 0 : out << " // Restore original workdir" << endl;
4466 0 : out << " gSystem->ChangeDirectory(cdir);" << endl;
4467 0 : out << " return kTRUE;" << endl;
4468 0 : out << "}" << endl;
4469 : }
4470 0 : Info("WriteAnalysisMacro", "\n##### Analysis macro to run on worker nodes <%s> written",fAnalysisMacro.Data());
4471 0 : }
4472 : Bool_t copy = kTRUE;
4473 0 : if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
4474 0 : if (copy) {
4475 0 : CdWork();
4476 0 : TString workdir = gGrid->GetHomeDirectory();
4477 0 : workdir += fGridWorkingDir;
4478 0 : if (FileExists(fAnalysisMacro)) gGrid->Rm(fAnalysisMacro);
4479 0 : Info("WriteAnalysisMacro", "\n##### Copying analysis macro: <%s> to your alien workspace", fAnalysisMacro.Data());
4480 : // TFile::Cp(Form("file:%s",fAnalysisMacro.Data()), Form("alien://%s/%s", workdir.Data(), fAnalysisMacro.Data()));
4481 0 : if (!copyLocal2Alien("WriteAnalysisMacro",fAnalysisMacro.Data(),
4482 0 : Form("alien://%s/%s", workdir.Data(),
4483 0 : fAnalysisMacro.Data()))) Fatal("","Terminating");
4484 0 : }
4485 0 : }
4486 :
4487 : //______________________________________________________________________________
4488 : void AliAnalysisAlien::WriteMergingMacro()
4489 : {
4490 : // Write a macro to merge the outputs per master job.
4491 0 : if (!fMergeViaJDL) return;
4492 0 : if (!fOutputFiles.Length()) {
4493 0 : Error("WriteMergingMacro", "No output file names defined. Are you running the right AliAnalysisAlien configuration ?");
4494 0 : return;
4495 : }
4496 0 : AliAnalysisManager *mgr = AliAnalysisManager::GetAnalysisManager();
4497 0 : TString mergingMacro = fExecutable;
4498 0 : mergingMacro.ReplaceAll(".sh","_merge.C");
4499 0 : if (gGrid && !fGridOutputDir.Contains("/")) fGridOutputDir = Form("%s/%s/%s", gGrid->GetHomeDirectory(), fGridWorkingDir.Data(), fGridOutputDir.Data());
4500 0 : if (!TestBit(AliAnalysisGrid::kSubmit)) {
4501 0 : ofstream out;
4502 0 : out.open(mergingMacro.Data(), ios::out);
4503 0 : if (!out.good()) {
4504 0 : Error("WriteMergingMacro", "could not open file %s for writing", fAnalysisMacro.Data());
4505 0 : return;
4506 : }
4507 : Bool_t hasSTEERBase = kFALSE;
4508 : Bool_t hasESD = kFALSE;
4509 : Bool_t hasAOD = kFALSE;
4510 : Bool_t hasANALYSIS = kFALSE;
4511 : Bool_t hasOADB = kFALSE;
4512 : Bool_t hasANALYSISalice = kFALSE;
4513 : Bool_t hasCORRFW = kFALSE;
4514 0 : TString func = mergingMacro;
4515 0 : TString comment;
4516 0 : func.ReplaceAll(".C", "");
4517 0 : out << "void " << func.Data() << "(const char *dir, Int_t stage=0)" << endl;
4518 0 : out << "{" << endl;
4519 0 : out << "// Automatically generated merging macro executed in grid subjobs" << endl << endl;
4520 0 : out << " TStopwatch timer;" << endl;
4521 0 : out << " timer.Start();" << endl << endl;
4522 : // Reset existing include path
4523 0 : out << "// Reset existing include path and add current directory first in the search" << endl;
4524 0 : out << " gSystem->SetIncludePath(\"-I.\");" << endl;
4525 0 : if (!fExecutableCommand.Contains("aliroot")) {
4526 0 : out << "// load base root libraries" << endl;
4527 0 : out << " gSystem->Load(\"libTree\");" << endl;
4528 0 : out << " gSystem->Load(\"libGeom\");" << endl;
4529 0 : out << " gSystem->Load(\"libVMC\");" << endl;
4530 0 : out << " gSystem->Load(\"libPhysics\");" << endl << endl;
4531 0 : out << " gSystem->Load(\"libMinuit\");" << endl << endl;
4532 : }
4533 0 : if (fAdditionalRootLibs.Length()) {
4534 : // in principle libtree /lib geom libvmc etc. can go into this list, too
4535 0 : out << "// Add aditional libraries" << endl;
4536 0 : TObjArray *list = fAdditionalRootLibs.Tokenize(" ");
4537 0 : TIter next(list);
4538 : TObjString *str;
4539 0 : TString buf;
4540 0 : while((str=(TObjString*)next())) {
4541 0 : buf = str->GetString();
4542 0 : if (buf.Contains(".so") || buf.Contains(".dylib")) {
4543 0 : buf.ReplaceAll(".so", "");
4544 0 : buf.ReplaceAll(".dylib", "");
4545 0 : out << " gSystem->Load(\"" << buf.Data() << "\");" << endl;
4546 : }
4547 : }
4548 0 : if (list) delete list;
4549 0 : }
4550 0 : out << "// Load analysis framework libraries" << endl;
4551 0 : if (!fPackages) {
4552 0 : if (!fExecutableCommand.Contains("aliroot")) {
4553 0 : out << " gSystem->Load(\"libSTEERBase\");" << endl;
4554 0 : out << " gSystem->Load(\"libESD\");" << endl;
4555 0 : out << " gSystem->Load(\"libAOD\");" << endl;
4556 : }
4557 0 : out << " gSystem->Load(\"libANALYSIS\");" << endl;
4558 0 : out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4559 0 : out << " gSystem->Load(\"libOADB\");" << endl;
4560 0 : out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4561 : } else {
4562 0 : TIter next(fPackages);
4563 : TObject *obj;
4564 0 : TString pkgname;
4565 0 : TString setupPar = "AliAnalysisAlien::SetupPar";
4566 0 : while ((obj=next())) {
4567 0 : pkgname = obj->GetName();
4568 0 : if (pkgname == "STEERBase" ||
4569 0 : pkgname == "STEERBase.par") hasSTEERBase = kTRUE;
4570 0 : if (pkgname == "ESD" ||
4571 0 : pkgname == "ESD.par") hasESD = kTRUE;
4572 0 : if (pkgname == "AOD" ||
4573 0 : pkgname == "AOD.par") hasAOD = kTRUE;
4574 0 : if (pkgname == "ANALYSIS" ||
4575 0 : pkgname == "ANALYSIS.par") hasANALYSIS = kTRUE;
4576 0 : if (pkgname == "OADB" ||
4577 0 : pkgname == "OADB.par") hasOADB = kTRUE;
4578 0 : if (pkgname == "ANALYSISalice" ||
4579 0 : pkgname == "ANALYSISalice.par") hasANALYSISalice = kTRUE;
4580 0 : if (pkgname == "CORRFW" ||
4581 0 : pkgname == "CORRFW.par") hasCORRFW = kTRUE;
4582 : }
4583 0 : if (hasANALYSISalice) setupPar = "SetupPar";
4584 0 : if (!hasSTEERBase) out << " gSystem->Load(\"libSTEERBase\");" << endl;
4585 0 : else out << " if (!" << setupPar << "(\"STEERBase\")) return;" << endl;
4586 0 : if (!hasESD) out << " gSystem->Load(\"libESD\");" << endl;
4587 0 : else out << " if (!" << setupPar << "(\"ESD\")) return;" << endl;
4588 0 : if (!hasAOD) out << " gSystem->Load(\"libAOD\");" << endl;
4589 0 : else out << " if (!" << setupPar << "(\"AOD\")) return;" << endl;
4590 0 : out << " gSystem->Load(\"libOADB\");" << endl;
4591 0 : if (!hasANALYSIS) out << " gSystem->Load(\"libANALYSIS\");" << endl;
4592 0 : else out << " if (!" << setupPar << "(\"ANALYSIS\")) return;" << endl;
4593 0 : if (!hasANALYSISalice) out << " gSystem->Load(\"libANALYSISalice\");" << endl;
4594 0 : else out << " if (!" << setupPar << "(\"ANALYSISalice\")) return;" << endl;
4595 0 : if (!hasOADB) out << " gSystem->Load(\"libOADB\");" << endl;
4596 0 : else out << " if (!" << setupPar << "(\"OADB\")) return;" << endl;
4597 0 : if (!hasCORRFW) out << " gSystem->Load(\"libCORRFW\");" << endl << endl;
4598 0 : else out << " if (!" << setupPar << "(\"CORRFW\")) return;" << endl << endl;
4599 0 : out << "// Compile other par packages" << endl;
4600 0 : next.Reset();
4601 0 : while ((obj=next())) {
4602 0 : pkgname = obj->GetName();
4603 0 : if (pkgname == "STEERBase" ||
4604 0 : pkgname == "STEERBase.par" ||
4605 0 : pkgname == "ESD" ||
4606 0 : pkgname == "ESD.par" ||
4607 0 : pkgname == "AOD" ||
4608 0 : pkgname == "AOD.par" ||
4609 0 : pkgname == "ANALYSIS" ||
4610 0 : pkgname == "ANALYSIS.par" ||
4611 0 : pkgname == "OADB" ||
4612 0 : pkgname == "OADB.par" ||
4613 0 : pkgname == "ANALYSISalice" ||
4614 0 : pkgname == "ANALYSISalice.par" ||
4615 0 : pkgname == "CORRFW" ||
4616 0 : pkgname == "CORRFW.par") continue;
4617 0 : out << " if (!" << setupPar << "(\"" << obj->GetName() << "\")) return;" << endl;
4618 : }
4619 0 : }
4620 0 : out << "// include path" << endl;
4621 : // Get the include path from the interpreter and remove entries pointing to AliRoot
4622 0 : out << " TString intPath = gInterpreter->GetIncludePath();" << endl;
4623 0 : out << " TObjArray *listpaths = intPath.Tokenize(\" \");" << endl;
4624 0 : out << " TIter nextpath(listpaths);" << endl;
4625 0 : out << " TObjString *pname;" << endl;
4626 0 : out << " while ((pname=(TObjString*)nextpath())) {" << endl;
4627 0 : out << " TString current = pname->GetName();" << endl;
4628 0 : out << " if (current.Contains(\"AliRoot\") || current.Contains(\"ALICE_ROOT\")) continue;" << endl;
4629 0 : out << " gSystem->AddIncludePath(current);" << endl;
4630 0 : out << " }" << endl;
4631 0 : out << " if (listpaths) delete listpaths;" << endl;
4632 0 : if (fIncludePath.Length()) out << " gSystem->AddIncludePath(\"" << fIncludePath.Data() << "\");" << endl;
4633 0 : out << " gROOT->ProcessLine(\".include $ALICE_ROOT/include\");" << endl;
4634 0 : out << " printf(\"Include path: %s\\n\", gSystem->GetIncludePath());" << endl << endl;
4635 0 : if (fMCLoop && !fGeneratorLibs.IsNull()) {
4636 0 : out << "// MC generator libraries" << endl;
4637 0 : TObjArray *list = fGeneratorLibs.Tokenize(" ");
4638 0 : TIter next(list);
4639 : TObjString *str;
4640 0 : while((str=(TObjString*)next())) {
4641 0 : out << " gSystem->Load(\"" << str->GetName() << "\");" << endl;
4642 : }
4643 0 : delete list;
4644 0 : }
4645 0 : if (fAdditionalLibs.Length()) {
4646 0 : out << "// Add aditional AliRoot libraries" << endl;
4647 0 : TString additionalLibs = fAdditionalLibs;
4648 0 : additionalLibs.Strip();
4649 0 : if (additionalLibs.Length() && fFriendLibs.Length())
4650 0 : additionalLibs += " ";
4651 0 : additionalLibs += fFriendLibs;
4652 0 : TObjArray *list = additionalLibs.Tokenize(" ");
4653 0 : TIter next(list);
4654 : TObjString *str;
4655 0 : TString buf;
4656 0 : while((str=(TObjString*)next())) {
4657 0 : buf = str->GetString();
4658 0 : if (buf.Contains(".so") || buf.Contains(".dylib")) {
4659 0 : buf.ReplaceAll(".so", "");
4660 0 : buf.ReplaceAll(".dylib", "");
4661 0 : out << " gSystem->Load(\"" << buf.Data() << "\");" << endl;
4662 : }
4663 : }
4664 0 : if (list) delete list;
4665 0 : }
4666 0 : out << endl;
4667 0 : out << "// Analysis source to be compiled at runtime (if any)" << endl;
4668 0 : if (fAnalysisSource.Length()) {
4669 0 : TObjArray *list = fAnalysisSource.Tokenize(" ");
4670 0 : TIter next(list);
4671 : TObjString *str;
4672 0 : while((str=(TObjString*)next())) {
4673 0 : out << " gROOT->ProcessLine(\".L " << str->GetString().Data() << "+g\");" << endl;
4674 : }
4675 0 : if (list) delete list;
4676 0 : }
4677 0 : out << endl;
4678 :
4679 0 : if (fFastReadOption) {
4680 0 : Warning("WriteMergingMacro", "!!! You requested FastRead option. Using xrootd flags to reduce timeouts in the grid merging jobs. Note that this may skip some files that could be accessed !!!");
4681 0 : out << "// fast xrootd reading enabled" << endl;
4682 0 : out << " printf(\"!!! You requested FastRead option. Using xrootd flags to reduce timeouts. Note that this may skip some files that could be accessed !!!\");" << endl;
4683 0 : out << " gEnv->SetValue(\"XNet.ConnectTimeout\",50);" << endl;
4684 0 : out << " gEnv->SetValue(\"XNet.RequestTimeout\",50);" << endl;
4685 0 : out << " gEnv->SetValue(\"XNet.MaxRedirectCount\",2);" << endl;
4686 0 : out << " gEnv->SetValue(\"XNet.ReconnectTimeout\",50);" << endl;
4687 0 : out << " gEnv->SetValue(\"XNet.FirstConnectMaxCnt\",1);" << endl << endl;
4688 : }
4689 : // Change temp directory to current one
4690 0 : out << "// Connect to AliEn" << endl;
4691 0 : out << " if (!TGrid::Connect(\"alien://\")) return;" << endl;
4692 0 : out << "// Set temporary merging directory to current one" << endl;
4693 0 : out << " gSystem->Setenv(\"TMPDIR\", gSystem->pwd());" << endl << endl;
4694 0 : out << "// Set temporary compilation directory to current one" << endl;
4695 0 : out << " gSystem->SetBuildDir(gSystem->pwd(), kTRUE);" << endl << endl;
4696 0 : out << " TString outputDir = dir;" << endl;
4697 0 : if (IsMergeAOD())
4698 0 : out << " TString outputFiles = \"" << GetListOfFiles("outaod") << "\";" << endl;
4699 : else
4700 0 : out << " TString outputFiles = \"" << GetListOfFiles("out") << "\";" << endl;
4701 0 : out << " TString mergeExcludes = \"" << fMergeExcludes << " " << fRegisterExcludes << "\";" << endl;
4702 0 : out << " TObjArray *list = outputFiles.Tokenize(\",\");" << endl;
4703 0 : out << " TIter *iter = new TIter(list);" << endl;
4704 0 : out << " TObjString *str;" << endl;
4705 0 : out << " TString outputFile;" << endl;
4706 0 : out << " Bool_t merged = kTRUE;" << endl;
4707 0 : TString analysisFile = fExecutable;
4708 0 : analysisFile.ReplaceAll(".sh", ".root");
4709 0 : out << " AliAnalysisManager *mgr = AliAnalysisAlien::LoadAnalysisManager(\""
4710 0 : << analysisFile << "\");" << endl;
4711 0 : out << " if (!mgr) {" << endl;
4712 0 : out << " printf(\"ERROR: Analysis manager could not be extracted from file \");" << endl;
4713 0 : out << " return;" << endl;
4714 0 : out << " }" << endl;
4715 0 : if (IsLocalTest()) {
4716 0 : out << " printf(\"===================================\\n\");" << endl;
4717 0 : out << " printf(\"Testing merging...\\n\");" << endl;
4718 0 : out << " printf(\"===================================\\n\");" << endl;
4719 : }
4720 0 : out << " while((str=(TObjString*)iter->Next())) {" << endl;
4721 0 : out << " outputFile = str->GetString();" << endl;
4722 0 : out << " if (outputFile.Contains(\"*\")) continue;" << endl;
4723 0 : out << " Int_t index = outputFile.Index(\"@\");" << endl;
4724 0 : out << " if (index > 0) outputFile.Remove(index);" << endl;
4725 0 : out << " // Skip already merged outputs" << endl;
4726 0 : out << " if (!gSystem->AccessPathName(outputFile)) {" << endl;
4727 0 : out << " printf(\"Output file <%s> found. Not merging again.\\n\",outputFile.Data());" << endl;
4728 0 : out << " continue;" << endl;
4729 0 : out << " }" << endl;
4730 0 : out << " if (mergeExcludes.Contains(outputFile.Data())) continue;" << endl;
4731 0 : out << " merged = AliAnalysisAlien::MergeOutput(outputFile, outputDir, " << fMaxMergeFiles << ", stage);" << endl;
4732 0 : out << " if (!merged) {" << endl;
4733 0 : out << " printf(\"ERROR: Cannot merge %s\\n\", outputFile.Data());" << endl;
4734 0 : out << " return;" << endl;
4735 0 : out << " }" << endl;
4736 0 : out << " }" << endl;
4737 0 : if (mgr && mgr->IsCollectThroughput() && !IsLocalTest()) {
4738 0 : out << " TString infolog = \"" << mgr->GetFileInfoLog() << "\";" << endl;
4739 0 : out << " AliAnalysisAlien::MergeInfo(infolog, outputDir);" << endl;
4740 : }
4741 0 : out << " // all outputs merged, validate" << endl;
4742 0 : out << " ofstream out;" << endl;
4743 0 : out << " out.open(\"outputs_valid\", ios::out);" << endl;
4744 0 : out << " out.close();" << endl;
4745 0 : out << " // read the analysis manager from file" << endl;
4746 0 : if (IsLocalTest()) {
4747 0 : out << " printf(\"===================================\\n\");" << endl;
4748 0 : out << " printf(\"Testing Terminate()...\\n\");" << endl;
4749 0 : out << " printf(\"===================================\\n\");" << endl;
4750 : } else {
4751 0 : out << " if (!outputDir.Contains(\"Stage\")) return;" << endl;
4752 : }
4753 0 : out << " mgr->SetRunFromPath(mgr->GetRunFromAlienPath(dir));" << endl;
4754 0 : out << " mgr->SetSkipTerminate(kFALSE);" << endl;
4755 0 : out << " mgr->PrintStatus();" << endl;
4756 0 : if (mgr) {
4757 0 : if (mgr->GetDebugLevel()>3) {
4758 0 : out << " gEnv->SetValue(\"XNet.Debug\", \"1\");" << endl;
4759 : } else {
4760 0 : if (TestBit(AliAnalysisGrid::kTest))
4761 0 : out << " AliLog::SetGlobalLogLevel(AliLog::kWarning);" << endl;
4762 : else
4763 0 : out << " AliLog::SetGlobalLogLevel(AliLog::kError);" << endl;
4764 : }
4765 : }
4766 0 : out << " TTree *tree = NULL;" << endl;
4767 0 : out << " mgr->StartAnalysis(\"gridterminate\", tree);" << endl;
4768 0 : out << "}" << endl << endl;
4769 0 : if (hasANALYSISalice) {
4770 0 : out <<"//________________________________________________________________________________" << endl;
4771 0 : out << "Bool_t SetupPar(const char *package) {" << endl;
4772 0 : out << "// Compile the package and set it up." << endl;
4773 0 : out << " TString pkgdir = package;" << endl;
4774 0 : out << " pkgdir.ReplaceAll(\".par\",\"\");" << endl;
4775 0 : out << " gSystem->Exec(TString::Format(\"tar xvzf %s.par\", pkgdir.Data()));" << endl;
4776 0 : out << " TString cdir = gSystem->WorkingDirectory();" << endl;
4777 0 : out << " gSystem->ChangeDirectory(pkgdir);" << endl;
4778 0 : out << " // Check for BUILD.sh and execute" << endl;
4779 0 : out << " if (!gSystem->AccessPathName(\"PROOF-INF/BUILD.sh\")) {" << endl;
4780 0 : out << " printf(\"*******************************\\n\");" << endl;
4781 0 : out << " printf(\"*** Building PAR archive ***\\n\");" << endl;
4782 0 : out << " printf(\"*******************************\\n\");" << endl;
4783 0 : out << " if (gSystem->Exec(\"PROOF-INF/BUILD.sh\")) {" << endl;
4784 0 : out << " ::Error(\"SetupPar\", \"Cannot build par archive %s\", pkgdir.Data());" << endl;
4785 0 : out << " gSystem->ChangeDirectory(cdir);" << endl;
4786 0 : out << " return kFALSE;" << endl;
4787 0 : out << " }" << endl;
4788 0 : out << " } else {" << endl;
4789 0 : out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/BUILD.sh for package %s\", pkgdir.Data());" << endl;
4790 0 : out << " gSystem->ChangeDirectory(cdir);" << endl;
4791 0 : out << " return kFALSE;" << endl;
4792 0 : out << " }" << endl;
4793 0 : out << " // Check for SETUP.C and execute" << endl;
4794 0 : out << " if (!gSystem->AccessPathName(\"PROOF-INF/SETUP.C\")) {" << endl;
4795 0 : out << " printf(\"*******************************\\n\");" << endl;
4796 0 : out << " printf(\"*** Setup PAR archive ***\\n\");" << endl;
4797 0 : out << " printf(\"*******************************\\n\");" << endl;
4798 0 : out << " gROOT->Macro(\"PROOF-INF/SETUP.C\");" << endl;
4799 0 : out << " } else {" << endl;
4800 0 : out << " ::Error(\"SetupPar\",\"Cannot access PROOF-INF/SETUP.C for package %s\", pkgdir.Data());" << endl;
4801 0 : out << " gSystem->ChangeDirectory(cdir);" << endl;
4802 0 : out << " return kFALSE;" << endl;
4803 0 : out << " }" << endl;
4804 0 : out << " // Restore original workdir" << endl;
4805 0 : out << " gSystem->ChangeDirectory(cdir);" << endl;
4806 0 : out << " return kTRUE;" << endl;
4807 0 : out << "}" << endl;
4808 : }
4809 0 : }
4810 : Bool_t copy = kTRUE;
4811 0 : if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
4812 0 : if (copy) {
4813 0 : CdWork();
4814 0 : TString workdir = gGrid->GetHomeDirectory();
4815 0 : workdir += fGridWorkingDir;
4816 0 : if (FileExists(mergingMacro)) gGrid->Rm(mergingMacro);
4817 0 : Info("WriteMergingMacro", "\n##### Copying merging macro: <%s> to your alien workspace", mergingMacro.Data());
4818 : // TFile::Cp(Form("file:%s",mergingMacro.Data()), Form("alien://%s/%s", workdir.Data(), mergingMacro.Data()));
4819 0 : if (!copyLocal2Alien("WriteMergeMacro",mergingMacro.Data(),
4820 0 : Form("%s/%s", workdir.Data(), mergingMacro.Data()))) Fatal("","Terminating");
4821 0 : }
4822 0 : }
4823 :
4824 : //______________________________________________________________________________
4825 : Bool_t AliAnalysisAlien::SetupPar(const char *package)
4826 : {
4827 : // Compile the par file archive pointed by <package>. This must be present in the current directory.
4828 : // Note that for loading the compiled library. The current directory should have precedence in
4829 : // LD_LIBRARY_PATH
4830 0 : TString pkgdir = package;
4831 0 : pkgdir.ReplaceAll(".par","");
4832 0 : gSystem->Exec(TString::Format("tar xzf %s.par", pkgdir.Data()));
4833 0 : TString cdir = gSystem->WorkingDirectory();
4834 0 : gSystem->ChangeDirectory(pkgdir);
4835 : // Check for BUILD.sh and execute
4836 0 : if (!gSystem->AccessPathName("PROOF-INF/BUILD.sh")) {
4837 0 : printf("**************************************************\n");
4838 0 : printf("*** Building PAR archive %s\n", package);
4839 0 : printf("**************************************************\n");
4840 0 : if (gSystem->Exec("PROOF-INF/BUILD.sh")) {
4841 0 : ::Error("SetupPar", "Cannot build par archive %s", pkgdir.Data());
4842 0 : gSystem->ChangeDirectory(cdir);
4843 0 : return kFALSE;
4844 : }
4845 : } else {
4846 0 : ::Error("SetupPar","Cannot access PROOF-INF/BUILD.sh for package %s", pkgdir.Data());
4847 0 : gSystem->ChangeDirectory(cdir);
4848 0 : return kFALSE;
4849 : }
4850 : // Check for SETUP.C and execute
4851 0 : if (!gSystem->AccessPathName("PROOF-INF/SETUP.C")) {
4852 0 : printf("**************************************************\n");
4853 0 : printf("*** Setup PAR archive %s\n", package);
4854 0 : printf("**************************************************\n");
4855 0 : gROOT->Macro("PROOF-INF/SETUP.C");
4856 0 : printf("*** Loaded library: %s\n", gSystem->GetLibraries(pkgdir,"",kFALSE));
4857 : } else {
4858 0 : ::Error("SetupPar","Cannot access PROOF-INF/SETUP.C for package %s", pkgdir.Data());
4859 0 : gSystem->ChangeDirectory(cdir);
4860 0 : return kFALSE;
4861 : }
4862 : // Restore original workdir
4863 0 : gSystem->ChangeDirectory(cdir);
4864 0 : return kTRUE;
4865 0 : }
4866 :
4867 : //______________________________________________________________________________
4868 : void AliAnalysisAlien::WriteExecutable()
4869 : {
4870 : // Generate the alien executable script.
4871 : // Patch executable with -x to catch error code
4872 0 : if (fExecutableCommand.Contains("root") &&
4873 0 : fExecutableCommand.Contains("-q") &&
4874 0 : !fExecutableCommand.Contains("-x")) fExecutableCommand += " -x";
4875 0 : if (!TestBit(AliAnalysisGrid::kSubmit)) {
4876 0 : ofstream out;
4877 0 : out.open(fExecutable.Data(), ios::out);
4878 0 : if (out.bad()) {
4879 0 : Error("WriteExecutable", "Bad file name for executable: %s", fExecutable.Data());
4880 0 : return;
4881 : }
4882 0 : out << "#!/bin/bash" << endl;
4883 : // Make sure we can properly compile par files
4884 0 : out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
4885 0 : out << "echo \"=========================================\"" << endl;
4886 0 : out << "echo \"############## PATH : ##############\"" << endl;
4887 0 : out << "echo $PATH" << endl;
4888 0 : out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4889 0 : out << "echo $LD_LIBRARY_PATH" << endl;
4890 0 : out << "echo \"############## ROOTSYS : ##############\"" << endl;
4891 0 : out << "echo $ROOTSYS" << endl;
4892 0 : out << "echo \"############## which root : ##############\"" << endl;
4893 0 : out << "which root" << endl;
4894 0 : out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4895 0 : out << "echo $ALICE_ROOT" << endl;
4896 0 : out << "echo \"############## which aliroot : ##############\"" << endl;
4897 0 : out << "which aliroot" << endl;
4898 0 : out << "echo \"############## system limits : ##############\"" << endl;
4899 0 : out << "ulimit -a" << endl;
4900 0 : out << "echo \"############## memory : ##############\"" << endl;
4901 0 : out << "free 2> /dev/null || { [[ `uname` == Darwin ]] && top -l 1 -s 0 | head -8 | tail -3; }" << endl;
4902 0 : out << "echo \"=========================================\"" << endl << endl;
4903 0 : out << fExecutableCommand << " ";
4904 0 : out << fAnalysisMacro.Data() << " " << fExecutableArgs.Data() << endl;
4905 0 : out << "RET=$?" << endl;
4906 0 : out << "if [ \"$RET\" != \"0\" ];then" << endl;
4907 0 : out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4908 0 : out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4909 0 : out << " let sig=\"$RET - 128\""<<endl;
4910 0 : out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4911 0 : out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4912 0 : out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4913 0 : out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4914 0 : out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4915 0 : out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4916 0 : out << " fi"<<endl;
4917 0 : out << " exit $RET"<< endl;
4918 0 : out << "fi" << endl << endl ;
4919 0 : out << "echo \"======== " << fAnalysisMacro.Data() << " finished with exit code: $RET ========\"" << endl;
4920 0 : out << "echo \"############## memory after: ##############\"" << endl;
4921 0 : out << "free -m" << endl;
4922 0 : }
4923 : Bool_t copy = kTRUE;
4924 0 : if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
4925 0 : if (copy) {
4926 0 : CdWork();
4927 0 : TString workdir = gGrid->GetHomeDirectory();
4928 0 : workdir += fGridWorkingDir;
4929 0 : TString executable = TString::Format("%s/%s", workdir.Data(), fExecutable.Data());
4930 0 : if (FileExists(executable)) gGrid->Rm(executable);
4931 0 : Info("WriteExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", fExecutable.Data());
4932 0 : if (!copyLocal2Alien("WriteExecutable",fExecutable.Data(),
4933 0 : executable.Data())) Fatal("","Terminating");
4934 0 : }
4935 0 : }
4936 :
4937 : //______________________________________________________________________________
4938 : void AliAnalysisAlien::WriteMergeExecutable()
4939 : {
4940 : // Generate the alien executable script for the merging job.
4941 0 : if (!fMergeViaJDL) return;
4942 0 : TString mergeExec = fExecutable;
4943 0 : mergeExec.ReplaceAll(".sh", "_merge.sh");
4944 0 : if (!TestBit(AliAnalysisGrid::kSubmit)) {
4945 0 : ofstream out;
4946 0 : out.open(mergeExec.Data(), ios::out);
4947 0 : if (out.bad()) {
4948 0 : Error("WriteMergingExecutable", "Bad file name for executable: %s", mergeExec.Data());
4949 0 : return;
4950 : }
4951 0 : out << "#!/bin/bash" << endl;
4952 : // Make sure we can properly compile par files
4953 0 : out << "export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH" << endl;
4954 0 : out << "echo \"=========================================\"" << endl;
4955 0 : out << "echo \"############## PATH : ##############\"" << endl;
4956 0 : out << "echo $PATH" << endl;
4957 0 : out << "echo \"############## LD_LIBRARY_PATH : ##############\"" << endl;
4958 0 : out << "echo $LD_LIBRARY_PATH" << endl;
4959 0 : out << "echo \"############## ROOTSYS : ##############\"" << endl;
4960 0 : out << "echo $ROOTSYS" << endl;
4961 0 : out << "echo \"############## which root : ##############\"" << endl;
4962 0 : out << "which root" << endl;
4963 0 : out << "echo \"############## ALICE_ROOT : ##############\"" << endl;
4964 0 : out << "echo $ALICE_ROOT" << endl;
4965 0 : out << "echo \"############## which aliroot : ##############\"" << endl;
4966 0 : out << "which aliroot" << endl;
4967 0 : out << "echo \"############## system limits : ##############\"" << endl;
4968 0 : out << "ulimit -a" << endl;
4969 0 : out << "echo \"############## memory : ##############\"" << endl;
4970 0 : out << "free -m" << endl;
4971 0 : out << "echo \"=========================================\"" << endl << endl;
4972 0 : TString mergeMacro = fExecutable;
4973 0 : mergeMacro.ReplaceAll(".sh", "_merge.C");
4974 0 : if (IsOneStageMerging())
4975 0 : out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\")\"" << endl;
4976 : else
4977 0 : out << "export ARG=\"" << mergeMacro << "(\\\"$1\\\",$2)\"" << endl;
4978 0 : out << fExecutableCommand << " " << "$ARG" << endl;
4979 0 : out << "RET=$?" << endl;
4980 0 : out << "if [ \"$RET\" != \"0\" ];then" << endl;
4981 0 : out << " echo \"======== ERROR : " << fAnalysisMacro.Data() << " finished with NON zero code: $RET ========\"" << endl;
4982 0 : out << " if [ \"$RET\" -gt 128 ] && [ \"$RET\" -lt 160 ]; then"<<endl;
4983 0 : out << " let sig=\"$RET - 128\""<<endl;
4984 0 : out << " sigs='HUP INT QUIT ILL TRAP ABRT BUS FPE"<<endl;
4985 0 : out << " KILL USR1 SEGV USR2 PIPE ALRM TERM STKFLT"<<endl;
4986 0 : out << " CHLD CONT STOP TSTP TTIN TTOU URG XCPU"<<endl;
4987 0 : out << " XFSZ VTALRM PROF WINCH IO PWR SYS'"<<endl;
4988 0 : out << " sig=SIG`echo $sigs | awk '{ print $'\"$sig\"' }'`"<<endl;
4989 0 : out << " echo \"======== it appears to have been killed with signal: $sig ========\""<<endl;
4990 0 : out << " fi"<<endl;
4991 0 : out << " exit $RET"<< endl;
4992 0 : out << "fi" << endl << endl ;
4993 0 : out << "echo \"======== " << mergeMacro.Data() << " finished with exit code: $? ========\"" << endl;
4994 0 : out << "echo \"############## memory after: ##############\"" << endl;
4995 0 : out << "free -m" << endl;
4996 0 : }
4997 : Bool_t copy = kTRUE;
4998 0 : if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
4999 0 : if (copy) {
5000 0 : CdWork();
5001 0 : TString workdir = gGrid->GetHomeDirectory();
5002 0 : workdir += fGridWorkingDir;
5003 0 : TString executable = TString::Format("%s/%s", workdir.Data(), mergeExec.Data());
5004 0 : if (FileExists(executable)) gGrid->Rm(executable);
5005 0 : Info("WriteMergeExecutable", "\n##### Copying executable file <%s> to your AliEn bin directory", mergeExec.Data());
5006 0 : if (!copyLocal2Alien("WriteMergeExecutable",mergeExec.Data(),
5007 0 : executable.Data())) Fatal("","Terminating");
5008 0 : }
5009 0 : }
5010 :
5011 : //______________________________________________________________________________
5012 : void AliAnalysisAlien::WriteProductionFile(const char *filename) const
5013 : {
5014 : // Write the production file to be submitted by LPM manager. The format is:
5015 : // First line: full_path_to_jdl estimated_no_subjobs_per_master
5016 : // Next lines: full_path_to_dataset XXX (XXX is a string)
5017 : // To submit, one has to: submit jdl XXX for all lines
5018 0 : ofstream out;
5019 0 : out.open(filename, ios::out);
5020 0 : if (out.bad()) {
5021 0 : Error("WriteProductionFile", "Bad file name: %s", filename);
5022 0 : return;
5023 : }
5024 0 : TString workdir;
5025 0 : if (!fProductionMode && !fGridWorkingDir.BeginsWith("/alice"))
5026 0 : workdir = gGrid->GetHomeDirectory();
5027 0 : workdir += fGridWorkingDir;
5028 0 : Int_t njobspermaster = 1000*fNrunsPerMaster/fSplitMaxInputFileNumber;
5029 0 : TString locjdl = Form("%s/%s", workdir.Data(),fJDLName.Data());
5030 0 : out << locjdl << " " << njobspermaster << endl;
5031 0 : Int_t nmasterjobs = fInputFiles->GetEntries();
5032 0 : for (Int_t i=0; i<nmasterjobs; i++) {
5033 0 : TString runOutDir = gSystem->BaseName(fInputFiles->At(i)->GetName());
5034 0 : runOutDir.ReplaceAll(".xml", "");
5035 0 : if (fOutputToRunNo)
5036 0 : out << Form("%s", fInputFiles->At(i)->GetName()) << " " << runOutDir << endl;
5037 : else
5038 0 : out << Form("%s", fInputFiles->At(i)->GetName()) << " " << Form("%03d", i) << endl;
5039 0 : }
5040 0 : if (gGrid) {
5041 0 : Info("WriteProductionFile", "\n##### Copying production file <%s> to your work directory", filename);
5042 0 : if (FileExists(filename)) gGrid->Rm(filename);
5043 : // TFile::Cp(Form("file:%s",filename), Form("alien://%s/%s", workdir.Data(),filename));
5044 0 : if (!copyLocal2Alien("WriteProductionFile", filename,
5045 0 : Form("%s/%s", workdir.Data(),filename))) Fatal("","Terminating");
5046 : }
5047 0 : }
5048 :
5049 : //______________________________________________________________________________
5050 : void AliAnalysisAlien::WriteValidationScript(Bool_t merge)
5051 : {
5052 : // Generate the alien validation script.
5053 : // Generate the validation script
5054 : TObjString *os;
5055 0 : if (fValidationScript.IsNull()) {
5056 0 : fValidationScript = fExecutable;
5057 0 : fValidationScript.ReplaceAll(".sh", "_validation.sh");
5058 0 : }
5059 0 : TString validationScript = fValidationScript;
5060 0 : if (merge) validationScript.ReplaceAll(".sh", "_merge.sh");
5061 0 : if (!Connect()) {
5062 0 : Error("WriteValidationScript", "Alien connection required");
5063 0 : return;
5064 : }
5065 0 : if (!fTerminateFiles.IsNull()) {
5066 0 : fTerminateFiles.Strip();
5067 0 : fTerminateFiles.ReplaceAll(" ",",");
5068 : }
5069 0 : TString outStream = "";
5070 0 : if (!TestBit(AliAnalysisGrid::kTest)) outStream = " >> stdout";
5071 0 : if (!TestBit(AliAnalysisGrid::kSubmit)) {
5072 0 : ofstream out;
5073 0 : out.open(validationScript, ios::out);
5074 0 : out << "#!/bin/bash" << endl;
5075 0 : out << "##################################################" << endl;
5076 0 : out << "validateout=`dirname $0`" << endl;
5077 0 : out << "validatetime=`date`" << endl;
5078 0 : out << "validated=\"0\";" << endl;
5079 0 : out << "error=0" << endl;
5080 0 : out << "if [ -z $validateout ]" << endl;
5081 0 : out << "then" << endl;
5082 0 : out << " validateout=\".\"" << endl;
5083 0 : out << "fi" << endl << endl;
5084 0 : out << "cd $validateout;" << endl;
5085 0 : out << "validateworkdir=`pwd`;" << endl << endl;
5086 0 : out << "echo \"*******************************************************\"" << outStream << endl;
5087 0 : out << "echo \"* Automatically generated validation script *\"" << outStream << endl;
5088 0 : out << "" << endl;
5089 0 : out << "echo \"* Time: $validatetime \"" << outStream << endl;
5090 0 : out << "echo \"* Dir: $validateout\"" << outStream << endl;
5091 0 : out << "echo \"* Workdir: $validateworkdir\"" << outStream << endl;
5092 0 : out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
5093 0 : out << "ls -la ./" << outStream << endl;
5094 0 : out << "echo \"* ----------------------------------------------------*\"" << outStream << endl << endl;
5095 0 : out << "##################################################" << endl;
5096 0 : out << "" << endl;
5097 :
5098 0 : out << "if [ ! -f stderr ] ; then" << endl;
5099 0 : out << " error=1" << endl;
5100 0 : out << " echo \"* ########## Job not validated - no stderr ###\" " << outStream << endl;
5101 0 : out << " echo \"Error = $error\" " << outStream << endl;
5102 0 : out << "fi" << endl;
5103 :
5104 0 : out << "parArch=`grep -Ei \"Cannot Build the PAR Archive\" stderr`" << endl;
5105 0 : out << "segViol=`grep -Ei \"Segmentation violation\" stderr`" << endl;
5106 0 : out << "segFault=`grep -Ei \"Segmentation fault\" stderr`" << endl;
5107 0 : out << "glibcErr=`grep -Ei '\\*\\*\\* glibc detected \\*\\*\\*' stderr`" << endl;
5108 0 : out << "" << endl;
5109 :
5110 0 : out << "if [ \"$parArch\" != \"\" ] ; then" << endl;
5111 0 : out << " error=1" << endl;
5112 0 : out << " echo \"* ########## Job not validated - PAR archive not built ###\" " << outStream << endl;
5113 0 : out << " echo \"$parArch\" " << outStream << endl;
5114 0 : out << " echo \"Error = $error\" " << outStream << endl;
5115 0 : out << "fi" << endl;
5116 :
5117 0 : out << "if [ \"$segViol\" != \"\" ] ; then" << endl;
5118 0 : out << " error=1" << endl;
5119 0 : out << " echo \"* ########## Job not validated - Segment. violation ###\" " << outStream << endl;
5120 0 : out << " echo \"$segViol\" " << outStream << endl;
5121 0 : out << " echo \"Error = $error\" " << outStream << endl;
5122 0 : out << "fi" << endl;
5123 :
5124 0 : out << "if [ \"$segFault\" != \"\" ] ; then" << endl;
5125 0 : out << " error=1" << endl;
5126 0 : out << " echo \"* ########## Job not validated - Segment. fault ###\" " << outStream << endl;
5127 0 : out << " echo \"$segFault\" " << outStream << endl;
5128 0 : out << " echo \"Error = $error\" " << outStream << endl;
5129 0 : out << "fi" << endl;
5130 :
5131 0 : out << "if [ \"$glibcErr\" != \"\" ] ; then" << endl;
5132 0 : out << " error=1" << endl;
5133 0 : out << " echo \"* ########## Job not validated - *** glibc detected *** ###\" " << outStream << endl;
5134 0 : out << " echo \"$glibcErr\" " << outStream << endl;
5135 0 : out << " echo \"Error = $error\" " << outStream << endl;
5136 0 : out << "fi" << endl;
5137 :
5138 : // Part dedicated to the specific analyses running into the train
5139 :
5140 0 : TString outputFiles = fOutputFiles;
5141 0 : if (merge && !fTerminateFiles.IsNull()) {
5142 0 : if (!outputFiles.IsNull()) outputFiles += ",";
5143 0 : outputFiles += fTerminateFiles;
5144 : }
5145 0 : TObjArray *arr = outputFiles.Tokenize(",");
5146 0 : TIter next1(arr);
5147 0 : TString outputFile;
5148 0 : while (!merge && (os=(TObjString*)next1())) {
5149 : // No need to validate outputs produced by merging since the merging macro does this
5150 0 : outputFile = os->GetString();
5151 0 : Int_t index = outputFile.Index("@");
5152 0 : if (index > 0) outputFile.Remove(index);
5153 0 : if (fTerminateFiles.Contains(outputFile)) continue;
5154 0 : if (outputFile.Contains("*")) continue;
5155 0 : out << "if ! [ -f " << outputFile.Data() << " ] ; then" << endl;
5156 0 : out << " error=1" << endl;
5157 0 : out << " echo \"Output file " << outputFile << " not found. Job FAILED !\"" << outStream << endl;
5158 0 : out << " echo \"Output file " << outputFile << " not found. Job FAILED !\" >> stderr" << endl;
5159 0 : out << "fi" << endl;
5160 0 : }
5161 0 : delete arr;
5162 0 : out << "if ! [ -f outputs_valid ] ; then" << endl;
5163 0 : out << " error=1" << endl;
5164 0 : out << " echo \"Output files were not validated by the analysis manager\" >> stdout" << endl;
5165 0 : out << " echo \"Output files were not validated by the analysis manager\" >> stderr" << endl;
5166 0 : out << "fi" << endl;
5167 :
5168 0 : out << "if [ $error = 0 ] ; then" << endl;
5169 0 : out << " echo \"* ---------------- Job Validated ------------------*\"" << outStream << endl;
5170 0 : if (!IsKeepLogs()) {
5171 0 : out << " echo \"* === Logs std* will be deleted === \"" << endl;
5172 0 : outStream = "";
5173 0 : out << " rm -f std*" << endl;
5174 : }
5175 0 : out << "fi" << endl;
5176 :
5177 0 : out << "echo \"* ----------------------------------------------------*\"" << outStream << endl;
5178 0 : out << "echo \"*******************************************************\"" << outStream << endl;
5179 0 : out << "cd -" << endl;
5180 0 : out << "exit $error" << endl;
5181 0 : }
5182 : Bool_t copy = kTRUE;
5183 0 : if (fProductionMode || TestBit(AliAnalysisGrid::kOffline) || TestBit(AliAnalysisGrid::kTest)) copy = kFALSE;
5184 0 : if (copy) {
5185 0 : CdWork();
5186 0 : TString workdir = gGrid->GetHomeDirectory();
5187 0 : workdir += fGridWorkingDir;
5188 0 : Info("WriteValidationScript", "\n##### Copying validation script <%s> to your AliEn working space", validationScript.Data());
5189 0 : if (FileExists(validationScript)) gGrid->Rm(validationScript);
5190 : // TFile::Cp(Form("file:%s",validationScript.Data()), Form("alien://%s/%s", workdir.Data(),validationScript.Data()));
5191 0 : if (!copyLocal2Alien("WriteValidationScript", validationScript.Data(),
5192 0 : Form("%s/%s",workdir.Data(), validationScript.Data()))) Fatal("","Terminating");
5193 0 : }
5194 0 : }
|
