Line data Source code
1 : #ifndef ALIANALYSISALIEN_H
2 : #define ALIANALYSISALIEN_H
3 : /* Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
4 : * See cxx source for full Copyright notice */
5 :
6 : // Author: Mihaela Gheata, 01/09/2008
7 :
8 : //==============================================================================
9 : // AliAnalysisAlien - AliEn utility class. Provides interface for creating
10 : // a personalized JDL, finding and creating a dataset.
11 : //==============================================================================
12 :
13 : #ifndef ALIANALYSISGRID_H
14 : #include "AliAnalysisGrid.h"
15 : #endif
16 :
17 : #ifndef ROOT_TString
18 : #include <TString.h>
19 : #endif
20 :
21 : #ifndef ROOT_TMath
22 : #include <TMath.h>
23 : #endif
24 :
25 : #ifndef ROOT_TMap
26 : #include <TMap.h>
27 : #endif
28 :
29 : class AliAnalysisManager;
30 : class AliAnalysisTaskCfg;
31 : class TGridJDL;
32 :
33 : class AliAnalysisAlien : public AliAnalysisGrid {
34 :
35 : public:
36 :
37 : AliAnalysisAlien();
38 : AliAnalysisAlien(const char *name);
39 : virtual ~AliAnalysisAlien();
40 : AliAnalysisAlien(const AliAnalysisAlien& other);
41 : AliAnalysisAlien& operator=(const AliAnalysisAlien& other);
42 : // Setters
43 : virtual void AddAdditionalLibrary(const char *name);
44 : virtual void AddIncludePath(const char *path);
45 : virtual void AddRunNumber(Int_t run);
46 : virtual void AddRunNumber(const char *run);
47 : virtual void AddRunList(const char *runList);
48 : virtual void AddDataFile(const char *lfn);
49 : virtual void AddExternalPackage(const char *name);
50 0 : virtual void SetExecutable(const char *name="analysis.sh") {fExecutable = name;}
51 0 : const char *GetExecutable() const {return fExecutable;}
52 0 : virtual void SetExecutableCommand(const char *command="root -b -q") {fExecutableCommand = command;}
53 0 : const char *GetExecutableCommand() const {return fExecutableCommand;}
54 0 : virtual void SetArguments(const char *name="") {fArguments = name;}
55 0 : const char *GetArguments() const {return fArguments;}
56 0 : virtual void SetExecutableArgs(const char *name="") {fExecutableArgs = name;}
57 0 : const char *GetExecutableArgs() const {return fExecutableArgs;}
58 0 : virtual void SetAnalysisMacro(const char *name="myAnalysis.C") {fAnalysisMacro = name;}
59 0 : const char *GetAnalysisMacro() const {return fAnalysisMacro;}
60 0 : virtual void SetAnalysisSource(const char *name="myAnalysisClass.cxx") {fAnalysisSource = name;}
61 0 : const char *GetAnalysisSource() const {return fAnalysisSource;}
62 0 : virtual void SetValidationScript(const char *name="validation.sh") {fValidationScript = name;}
63 0 : const char *GetValidationScript() const {return fValidationScript;}
64 0 : virtual void SetAdditionalLibs(const char *list) {fAdditionalLibs = list;}
65 0 : const char *GetAdditionalLibs() const {return fAdditionalLibs;}
66 0 : virtual void SetAdditionalRootLibs(const char *list) {fAdditionalRootLibs = list;}
67 0 : const char *GetAdditionalRootLibs() const {return fAdditionalRootLibs;}
68 0 : virtual void SetPrice(Int_t price=1) {fPrice = price;}
69 0 : Int_t GetPrice() const {return fPrice;}
70 0 : virtual void SetRunRange(Int_t min, Int_t max) {fRunRange[0] = min; fRunRange[1] = max;}
71 0 : void GetRunRange(Int_t &min, Int_t &max) {min = fRunRange[0]; max = fRunRange[1];}
72 0 : virtual void SetJobTag(const char *tag="") {fJobTag = tag;}
73 0 : const char *GetJobTag() const {return fJobTag;}
74 0 : virtual void SetNrunsPerMaster(Int_t nruns=1) {fNrunsPerMaster = nruns;}
75 0 : Int_t GetNrunsPerMaster() const {return fNrunsPerMaster;}
76 0 : virtual void SetMaxMergeFiles(Int_t nfiles) {fMaxMergeFiles = nfiles;}
77 0 : Int_t GetMaxMergeFiles() const {return fMaxMergeFiles;}
78 0 : virtual void SetMaxMergeStages(Int_t nstages) {fMaxMergeStages = nstages;}
79 0 : Int_t GetMaxMergeStages() const {return fMaxMergeStages;}
80 0 : virtual void SetSplitMode(const char *type="se") {fSplitMode = type;}
81 0 : const char *GetSplitMode() const {return fSplitMode;}
82 0 : virtual void SetSplitMaxInputFileNumber(Int_t nfiles=100) {fSplitMaxInputFileNumber = nfiles;}
83 0 : Int_t GetSplitMaxInputFileNumber() const {return fSplitMaxInputFileNumber;}
84 0 : virtual void SetAPIVersion(const char *version) {fAPIVersion = version;}
85 0 : const char *GetAPIVersion() const {return fAPIVersion;}
86 0 : virtual void SetROOTVersion(const char *version) {fROOTVersion = version;}
87 0 : const char *GetROOTVersion() const {return fROOTVersion;}
88 0 : virtual void SetAliROOTVersion(const char *version) {fAliROOTVersion=version;}
89 0 : const char *GetAliROOTVersion() const {return fAliROOTVersion;}
90 0 : virtual void SetAliPhysicsVersion(const char *version) {fAliPhysicsVersion=version;}
91 0 : const char *GetAliPhysicsVersion() const {return fAliPhysicsVersion;}
92 0 : virtual void SetUser(const char *user) {fUser = user;}
93 0 : const char *GetUser() const {return fUser;}
94 0 : virtual void SetTTL(Int_t ttl=30000) {fTTL = ttl;}
95 0 : Int_t GetTTL() const {return fTTL;}
96 0 : virtual void SetGridWorkingDir(const char *name="workdir") {fGridWorkingDir = name;}
97 0 : const char *GetGridWorkingDir() const {return fGridWorkingDir;}
98 0 : virtual void SetGridDataDir(const char *name) {fGridDataDir = name;}
99 0 : const char *GetGridDataDir() const {return fGridDataDir;}
100 0 : void SetGeneratorLibs(const char *libs) {fGeneratorLibs = libs;}
101 0 : const char *GetGeneratorLibs() const {return fGeneratorLibs;}
102 0 : virtual void SetDataPattern(const char *pattern="*AliESDs.root") {fDataPattern = pattern;}
103 0 : const char *GetDataPattern() const {return fDataPattern;}
104 : virtual void SetFriendChainName(const char *name="", const char *libnames="");
105 0 : const char *GetFriendChainName() const {return fFriendChainName;}
106 : virtual void SetDefaultOutputs(Bool_t flag);
107 0 : virtual void SetGridOutputDir(const char *name="output") {fGridOutputDir = name;}
108 0 : const char *GetGridOutputDir() const {return fGridOutputDir;}
109 : virtual void SetOutputArchive(const char *list="log_archive.zip:std*@disk=1 root_archive.zip:*.root@disk=2");
110 0 : const char *GetOutputArchive() const {return fOutputArchive;}
111 : virtual void SetOutputFiles(const char *list);
112 0 : const char *GetOutputFiles() const {return fOutputFiles;}
113 0 : virtual void SetOutputToRunNo(Int_t mode=1) {fOutputToRunNo = mode;}
114 0 : Int_t GetOutputToRunNoMode() const {return fOutputToRunNo;}
115 0 : virtual void SetInputFormat(const char *format="xml-single") {fInputFormat = format;}
116 0 : const char *GetInputFormat() const {return fInputFormat;}
117 0 : virtual void SetMaxInitFailed(Int_t nfail=5) {fMaxInitFailed = nfail;}
118 0 : Int_t GetMaxInitFailed() const {return fMaxInitFailed;}
119 0 : virtual void SetTerminateFiles(const char *list) {fTerminateFiles = list;}
120 0 : const char *GetTerminateFiles() const {return fTerminateFiles;}
121 0 : virtual void SetMergeExcludes(const char *list) {fMergeExcludes = list; fMergeExcludes.ReplaceAll(",", " "); }
122 0 : const char *GetMergeExcludes() const {return fMergeExcludes;}
123 0 : virtual void SetMergeViaJDL(Bool_t on=kTRUE) {fMergeViaJDL = on ? 1 : 0;}
124 0 : Bool_t IsMergeViaJDL() const {return fMergeViaJDL;}
125 0 : virtual void SetMergeDirName(const char *name) {fMergeDirName = name;}
126 0 : const char *GetMergeDirName() const {return fMergeDirName;}
127 0 : virtual void SetMasterResubmitThreshold(Int_t percentage) {fMasterResubmitThreshold = percentage;}
128 0 : Int_t GetMasterResubmitThreshold() const {return fMasterResubmitThreshold;}
129 0 : void SetMCLoop(Bool_t flag=kTRUE) {fMCLoop = flag;}
130 0 : virtual void SetNtestFiles(Int_t nfiles) {fNtestFiles = nfiles;}
131 0 : Int_t GetNtestFiles() const {return fNtestFiles;}
132 0 : virtual void SetNumberOfReplicas(Int_t ncopies) {fNreplicas = TMath::Min(ncopies,4);}
133 0 : Int_t GetNumberOfReplicas() const {return fNreplicas;}
134 0 : virtual void SetJDLName(const char *name="analysis.jdl") {fJDLName = name;}
135 0 : const char *GetJDLName() const {return fJDLName;}
136 0 : virtual void SetProductionMode(Int_t mode=1) {fProductionMode = mode;}
137 0 : Int_t GetProductionMode() const {return fProductionMode;}
138 0 : virtual void SetRegisterExcludes(const char *list) {fRegisterExcludes = list; fRegisterExcludes.ReplaceAll(",", " "); }
139 0 : const char *GetRegisterExcludes() const {return fRegisterExcludes;}
140 : virtual void SetRunPrefix(const char *prefix);
141 0 : const char *GetRunPrefix() const {return fRunPrefix;}
142 0 : virtual void SetOutputSingleFolder(const char *folder) {fOutputSingle = folder; fSplitMode="file"; fSplitMaxInputFileNumber=1;}
143 0 : const char *GetOutputSingleFolder() const {return fOutputSingle;}
144 0 : virtual void SetFastReadOption(Bool_t on=kTRUE) {fFastReadOption = on ? 1 : 0;}
145 0 : Bool_t IsFastReadOption() const {return fFastReadOption;}
146 0 : virtual void SetOverwriteMode(Bool_t on=kTRUE) {fOverwriteMode = on ? 1 : 0;}
147 0 : Bool_t IsOverwriteMode() const {return fOverwriteMode;}
148 0 : virtual void SetDropToShell(Bool_t drop=true) {fDropToShell = drop;}
149 0 : Bool_t IsDropToShell() const {return fDropToShell;}
150 0 : virtual void SetTreeName(const char *name) {fTreeName = name;}
151 0 : const char *GetTreeName() const {return fTreeName;}
152 :
153 0 : TGridJDL *GetGridJDL() const {return fGridJDL;}
154 0 : TGridJDL *GetMergingJDL() const {return fMergingJDL;}
155 0 : Int_t GetNMCevents() const {return fNMCevents;}
156 0 : Int_t GetNMCjobs() const {return fNMCjobs;}
157 0 : void SetNMCevents(Int_t nevents) {fNMCevents = nevents;}
158 0 : void SetNMCjobs(Int_t njobs) {fNMCjobs = njobs;}
159 : //Utilities
160 : void AddModule(AliAnalysisTaskCfg *module);
161 : void AddModules(TObjArray *list);
162 : AliAnalysisManager *CreateAnalysisManager(const char *name, const char *filename="");
163 : Int_t GetNmodules() const;
164 : AliAnalysisTaskCfg *GetModule(const char *name);
165 : Bool_t LoadModules();
166 : Bool_t LoadFriendLibs() const;
167 : Bool_t GenerateTest(const char *name, const char *modname="");
168 : Bool_t GenerateTrain(const char *name);
169 : virtual Bool_t CreateDataset(const char *pattern);
170 : Int_t CopyLocalDataset(const char *griddir, const char *pattern, Int_t nfiles, const char *output="data.txt", const char *archivefile="", const char *outputdir="data");
171 : virtual Bool_t CreateJDL();
172 : virtual void EnablePackage(const char *package);
173 : static Bool_t DirectoryExists(const char *lfn);
174 : static Bool_t FileExists(const char *lfn);
175 : static const char *GetJobStatus(Int_t jobidstart, Int_t lastid, Int_t &nrunning, Int_t &nwaiting, Int_t &nerror, Int_t &ndone);
176 : const char *GetListOfFiles(const char *type);
177 : Bool_t CheckMergedFiles(const char *filename, const char *aliendir, Int_t nperchunk, const char *jdl="");
178 : static AliAnalysisManager *LoadAnalysisManager(const char *fname);
179 : static Bool_t MergeInfo(const char *output, const char *collection);
180 : static Bool_t MergeOutput(const char *output, const char *basedir, Int_t nmaxmerge, Int_t stage=0);
181 : virtual Bool_t MergeOutputs();
182 : virtual void Print(Option_t *option="") const;
183 : static Long64_t RunMacroAndExtractLibs(const char* macro, const char *args, TString &libs);
184 : virtual Bool_t StartAnalysis(Long64_t nentries=123456789, Long64_t firstentry=0);
185 : static Bool_t SetupPar(const char *package);
186 : virtual Bool_t Submit();
187 : virtual Bool_t SubmitMerging();
188 : static Int_t SubmitSingleJob(const char *query);
189 : virtual void WriteAnalysisFile();
190 : virtual void WriteAnalysisMacro(Long64_t nentries=123456789, Long64_t firstentry=0);
191 : virtual void WriteMergingMacro();
192 : virtual void WriteMergeExecutable();
193 : virtual void WriteExecutable();
194 : virtual Bool_t WriteJDL(Bool_t copy);
195 : virtual void WriteProductionFile(const char *filename) const;
196 : virtual void WriteValidationScript(Bool_t merge=kFALSE);
197 :
198 : // PROOF mode
199 0 : virtual void SetProofCluster(const char *cluster) {fProofCluster = cluster;}
200 0 : virtual void SetProofDataSet(const char *dataset) {fProofDataSet = dataset;}
201 0 : virtual const char *GetProofDataSet() const {return fProofDataSet;}
202 : virtual void SetProofParameter(const char *pname, const char *value);
203 : const char *GetProofParameter(const char *pname) const;
204 0 : virtual void SetProofReset(Int_t mode) {fProofReset = mode;}
205 0 : virtual void SetNproofWorkers(Int_t nworkers) {fNproofWorkers = nworkers;}
206 0 : virtual void SetNproofWorkersPerSlave(Int_t nworkers) {fNproofWorkersPerSlave = nworkers;}
207 : virtual void SetRootVersionForProof(const char *version);
208 0 : virtual void SetAliRootMode(const char *mode) {fAliRootMode = mode;}
209 0 : virtual void SetProofProcessOpt(const char *proofOpt="") {fProofProcessOpt = proofOpt;}
210 0 : virtual TString GetProofProcessOpt() {return fProofProcessOpt;}
211 : // .txt file containing the list of files to be chained in test mode
212 0 : virtual void SetFileForTestMode(const char *filename) {fFileForTestMode = filename;}
213 0 : const char *GetFileForTestMode() const {return fFileForTestMode;}
214 : virtual TChain *GetChainForTestMode(const char *treeName) const;
215 0 : virtual const TString& GetGridJobIDs() const { return fGridJobIDs; }
216 0 : virtual const TString& GetGridStages() const { return fGridStages; }
217 : protected:
218 : void CdWork();
219 : Bool_t CheckInputData();
220 : void CheckDataType(const char *lfn, Bool_t &is_collection, Bool_t &is_xml, Bool_t &use_tags);
221 : virtual Bool_t Connect();
222 : virtual void SetDefaults();
223 : Bool_t SubmitNext();
224 :
225 : Bool_t IsCollection(const char *lfn) const;
226 : Bool_t IsMCLoop() const {return fMCLoop;}
227 : virtual Bool_t IsSingleOutput() const;
228 : Bool_t IsUsingTags() const {return TObject::TestBit(AliAnalysisGrid::kUseTags);}
229 : Bool_t LoadModule(AliAnalysisTaskCfg *mod);
230 : Bool_t CheckDependencies();
231 : Bool_t CheckFileCopy(const char *alienpath);
232 :
233 : private:
234 : TGridJDL *fGridJDL; //! JDL maker
235 : TGridJDL *fMergingJDL; //! JDL maker
236 : Int_t fPrice; // Grid price for the job;
237 : Int_t fTTL; // Time to live.
238 : Int_t fSplitMaxInputFileNumber; // Maximum number of files to be processed per subjob
239 : Int_t fMaxInitFailed; // Maximum initial consecutive subjobs accepted to fail
240 : Int_t fMasterResubmitThreshold; // Failed jobs will be resubmitted until this DONE ratio
241 : Int_t fNtestFiles; // Number of files used in the testing case
242 : Int_t fRunRange[2]; // Run range
243 : Int_t fNrunsPerMaster; // Number of runs per masterjob
244 : Int_t fMaxMergeFiles; // Maximum number of files to be merged in one chunk
245 : Int_t fMaxMergeStages; // Maximum number of merging stages
246 : Int_t fNsubmitted; // Number of jobs submitted
247 : Int_t fProductionMode; // Production mode (0-off, 1-on)
248 : Int_t fOutputToRunNo; // Use run number as output directory
249 : Int_t fMergeViaJDL; // Enable merging via automatic JDL
250 : Int_t fFastReadOption; // Use xrootd tweaks to reduce timeouts in file access
251 : Int_t fOverwriteMode; // Overwrite existing files if any
252 : Int_t fNreplicas; // Number of replicas for the output files
253 : Int_t fNproofWorkers; // Number of workers in proof mode
254 : Int_t fNproofWorkersPerSlave; // Max number of workers per slave in proof mode
255 : Int_t fProofReset; // Proof reset mode: 0=no reset, 1=soft, 2=hard
256 : Int_t fNMCevents; // Number of MC events in MC loop mode
257 : Int_t fNMCjobs; // Number of MC jobs in MC loop mode
258 : TString fRunNumbers; // List of runs to be processed
259 : TString fExecutable; // Executable script for AliEn job
260 : TString fExecutableCommand; // Command(s) to be executed in the executable script
261 : TString fArguments; // Arguments for the executable script
262 : TString fExecutableArgs; // arguments added to the executable script after the analysis macro
263 : TString fAnalysisMacro; // Root macro steering the analysis
264 : TString fAnalysisSource; // User analysis implementation (.cxx) file(s)
265 : TString fValidationScript; // Name of the validation script
266 : TString fAdditionalRootLibs; // List (separated by blacs) of additional libraries needed for/before analysis libs/par file compilation
267 : TString fAdditionalLibs; // List (separated by blacs) of additional libraries needed for the analysis loaded AFTER all par files
268 : TString fGeneratorLibs; // Extra libraries needed by the generator
269 : TString fSplitMode; // Job split mode
270 : TString fAPIVersion; // API version
271 : TString fROOTVersion; // ROOT version
272 : TString fAliROOTVersion; // AliROOT version
273 : TString fAliPhysicsVersion; // AliPhysics version
274 : TString fExternalPackages; // External packages
275 : TString fUser; // AliEn user name
276 : TString fGridWorkingDir; // AliEn directory containing the input packages
277 : TString fGridDataDir; // AliEn data production directory
278 : TString fDataPattern; // Data pattern for 'find' command
279 : TString fGridOutputDir; // AliEn directory (wrt work dir) where the output should be written
280 : TString fOutputArchive; // List of output archives separated by blancs
281 : TString fOutputFiles; // List of output files separated by blancs
282 : TString fInputFormat; // Input format (xml-single)
283 : TString fDatasetName; // Dataset xml file to be created
284 : TString fJDLName; // JDL file to be generated
285 : TString fTerminateFiles; // List of output files produced during Terminate
286 : TString fMergeExcludes; // List of output files excluded from merging
287 : TString fRegisterExcludes; // List of liles not to be registered/merged
288 : TString fIncludePath; // Include path
289 : TString fCloseSE; // Preffered storage element. Taken from alien_CLOSE_SE environment.
290 : TString fFriendChainName; // File name to construct friend chain (for AOD)
291 : TString fJobTag; // Job tag
292 : TString fOutputSingle; // Directory name for the output when split is per file
293 : TString fRunPrefix; // Run prefix to be applied to run numbers
294 : TString fProofCluster; // Proof cluster name
295 : TString fProofDataSet; // Proof dataset to be used
296 : TString fFileForTestMode; // .txt file for the chain to be used in PROOF test mode
297 : TString fAliRootMode; // AliRoot mode among the list supported by the proof cluster
298 : TString fProofProcessOpt; // Option passed to proof process
299 : TString fMergeDirName; // Name of the directory that should be added to the output directory
300 : TObjArray *fInputFiles; // List of input files to be processed by the job
301 : TObjArray *fPackages; // List of packages to be used
302 : TObjArray *fModules; // List of AliAnalysisTaskCfg modules
303 : TMap fProofParam; // Key-value pairs for proof mode
304 : Bool_t fDropToShell; // If true, execute aliensh on start
305 : Bool_t fMCLoop; // MC loop flag
306 : TString fGridJobIDs; // List of last committed jobs
307 : TString fGridStages; // List of last committed jobs
308 : TString fFriendLibs; // List of libs (separated by blacs) needed for friends processing
309 : TString fTreeName; // Name of the tree to be analyzed
310 :
311 170 : ClassDef(AliAnalysisAlien, 27) // Class providing some AliEn utilities
312 : };
313 : #endif
|
