View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  
18  package org.apache.commons.net.ftp.parser;
19  
20  import java.text.ParseException;
21  import java.util.List;
22  
23  import org.apache.commons.net.ftp.FTPClientConfig;
24  import org.apache.commons.net.ftp.FTPFile;
25  
26  /**
27   * Implementation of FTPFileEntryParser and FTPFileListParser for IBM zOS/MVS
28   * Systems.
29   * 
30   * @author <a href="henrik.sorensen@balcab.ch">Henrik Sorensen</a>
31   * @author <a href="jnadler@srcginc.com">Jeff Nadler</a>
32   * @author <a href="wnoto@openfinance.com">William Noto</a>
33   * 
34   * @version $Id: MVSFTPEntryParser.java 658520 2008-05-21 01:14:11Z sebb $
35   * @see org.apache.commons.net.ftp.FTPFileEntryParser FTPFileEntryParser (for
36   *      usage instructions)
37   */
38  public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl {
39  
40      static final int UNKNOWN_LIST_TYPE = -1;
41      static final int FILE_LIST_TYPE = 0;
42      static final int MEMBER_LIST_TYPE = 1;
43      static final int UNIX_LIST_TYPE = 2;
44      static final int JES_LEVEL_1_LIST_TYPE = 3;
45      static final int JES_LEVEL_2_LIST_TYPE = 4;
46  
47      private int isType = UNKNOWN_LIST_TYPE;
48  
49      /**
50       * Fallback parser for Unix-style listings
51       */
52      private UnixFTPEntryParser unixFTPEntryParser;
53  
54      /**
55       * Dates are ignored for file lists, but are used for member lists where
56       * possible
57       */
58      static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18
59                                                                      // 13:52
60  
61      /**
62       * Matches these entries: Volume Unit Referred Ext Used Recfm Lrecl BlkSz
63       * Dsorg Dsname B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK
64       * 
65       */
66      static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume
67                                                                  // ignored
68              "\\S+\\s+" + // unit - ignored
69              "\\S+\\s+" + // access date - ignored
70              "\\S+\\s+" + // extents -ignored
71              "\\S+\\s+" + // used - ignored
72              "[FV]\\S*\\s+" + // recfm - must start with F or V
73              "\\S+\\s+" + // logical record length -ignored
74              "\\S+\\s+" + // block size - ignored
75              "(PS|PO|PO-E)\\s+" + // Dataset organisation. Many exist
76              // but only support: PS, PO, PO-E
77              "(\\S+)\\s*"; // Dataset Name (file name)
78  
79      /**
80       * Matches these entries: Name VV.MM Created Changed Size Init Mod Id
81       * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001
82       */
83      static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name
84              "\\S+\\s+" + // version, modification (ignored)
85              "\\S+\\s+" + // create date (ignored)
86              "(\\S+)\\s+" + // modification date
87              "(\\S+)\\s+" + // modification time
88              "\\S+\\s+" + // size in lines (ignored)
89              "\\S+\\s+" + // size in lines at creation(ignored)
90              "\\S+\\s+" + // lines modified (ignored)
91              "\\S+\\s*"; // id of user who modified (ignored)
92  
93      /**
94       * Matches these entries, note: no header: IBMUSER1 JOB01906 OUTPUT 3 Spool
95       * Files 012345678901234567890123456789012345678901234 1 2 3 4
96       */
97      static final String JES_LEVEL_1_LIST_REGEX = "(\\S+)\\s+" + // job
98                                                                          // name
99                                                                          // ignored
100             "(\\S+)\\s+" + // job number
101             "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE)
102             "(\\S+)\\s+" + // number of spool files
103             "(\\S+)\\s+" + // Text "Spool" ignored
104             "(\\S+)\\s*" // Text "Files" ignored
105     ;
106 
107     /**
108      * JES INTERFACE LEVEL 2 parser Matches these entries: JOBNAME JOBID OWNER
109      * STATUS CLASS IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files
110      * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files
111      * 012345678901234567890123456789012345678901234 1 2 3 4
112      * 012345678901234567890123456789012345678901234567890
113      */
114 
115     static final String JES_LEVEL_2_LIST_REGEX = "(\\S+)\\s+" + // job
116                                                                         // name
117                                                                         // ignored
118             "(\\S+)\\s+" + // job number
119             "(\\S+)\\s+" + // owner ignored
120             "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored
121             "(\\S+)\\s+" + // job class ignored
122             "(\\S+).*" // rest ignored
123     ;
124 
125     /*
126      * ---------------------------------------------------------------------
127      * Very brief and incomplete description of the zOS/MVS-filesystem. (Note:
128      * "zOS" is the operating system on the mainframe, and is the new name for
129      * MVS)
130      * 
131      * The filesystem on the mainframe does not have hierarchal structure as for
132      * example the unix filesystem. For a more comprehensive description, please
133      * refer to the IBM manuals
134      * 
135      * @LINK:
136      * http://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS
137      * 
138      * 
139      * Dataset names =============
140      * 
141      * A dataset name consist of a number of qualifiers separated by '.', each
142      * qualifier can be at most 8 characters, and the total length of a dataset
143      * can be max 44 characters including the dots.
144      * 
145      * 
146      * Dataset organisation ====================
147      * 
148      * A dataset represents a piece of storage allocated on one or more disks.
149      * The structure of the storage is described with the field dataset
150      * organinsation (DSORG). There are a number of dataset organisations, but
151      * only two are usable for FTP transfer.
152      * 
153      * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E:
154      * extended partitioned dataset
155      * 
156      * The PS file is just a flat file, as you would find it on the unix file
157      * system.
158      * 
159      * The PO and PO-E files, can be compared to a single level directory
160      * structure. A PO file consist of a number of dataset members, or files if
161      * you will. It is possible to CD into the file, and to retrieve the
162      * individual members.
163      * 
164      * 
165      * Dataset record format =====================
166      * 
167      * The physical layout of the dataset is described on the dataset itself.
168      * There are a number of record formats (RECFM), but just a few is relavant
169      * for the FTP transfer.
170      * 
171      * Any one beginning with either F or V can safely used by FTP transfer. All
172      * others should only be used with great care, so this version will just
173      * ignore the other record formats. F means a fixed number of records per
174      * allocated storage, and V means a variable number of records.
175      * 
176      * 
177      * Other notes ===========
178      * 
179      * The file system supports automatically backup and retrieval of datasets.
180      * If a file is backed up, the ftp LIST command will return: ARCIVE Not
181      * Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST
182      * 
183      * 
184      * Implementation notes ====================
185      * 
186      * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning
187      * with F or V, is fully parsed.
188      * 
189      * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set.
190      * FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name
191      * FTPFile.Timestamp: change time or null
192      * 
193      * 
194      * 
195      * Additional information ======================
196      * 
197      * The MVS ftp server supports a number of features via the FTP interface.
198      * The features are controlled with the FTP command quote site filetype=<SEQ|JES|DB2>
199      * SEQ is the default and used for normal file transfer JES is used to
200      * interact with the Job Entry Subsystem (JES) similar to a job scheduler
201      * DB2 is used to interact with a DB2 subsystem
202      * 
203      * This parser supports SEQ and JES.
204      * 
205      * 
206      * 
207      * 
208      * 
209      * 
210      */
211 
212     /**
213      * The sole constructor for a MVSFTPEntryParser object.
214      * 
215      */
216     public MVSFTPEntryParser() {
217         super(""); // note the regex is set in preParse.
218         super.configure(null); // configure parser with default configurations
219     }
220 
221     /**
222      * Parses a line of an z/OS - MVS FTP server file listing and converts it
223      * into a usable format in the form of an <code> FTPFile </code> instance.
224      * If the file listing line doesn't describe a file, then
225      * <code> null </code> is returned. Otherwise a <code> FTPFile </code>
226      * instance representing the file is returned.
227      * 
228      * @param entry
229      *            A line of text from the file listing
230      * @return An FTPFile instance corresponding to the supplied entry
231      */
232     public FTPFile parseFTPEntry(String entry) {
233         boolean isParsed = false;
234         FTPFile f = new FTPFile();
235 
236         if (isType == FILE_LIST_TYPE)
237             isParsed = parseFileList(f, entry);
238         else if (isType == MEMBER_LIST_TYPE) {
239             isParsed = parseMemberList(f, entry);
240             if (!isParsed)
241                 isParsed = parseSimpleEntry(f, entry);
242         } else if (isType == UNIX_LIST_TYPE) {
243             isParsed = parseUnixList(f, entry);
244         } else if (isType == JES_LEVEL_1_LIST_TYPE) {
245             isParsed = parseJeslevel1List(f, entry);
246         } else if (isType == JES_LEVEL_2_LIST_TYPE) {
247             isParsed = parseJeslevel2List(f, entry);
248         }
249 
250         if (!isParsed)
251             f = null;
252         
253         return f;
254     }
255 
256     /**
257      * Parse entries representing a dataset list. Only datasets with DSORG PS or
258      * PO or PO-E and with RECFM F* or V* will be parsed.
259      * 
260      * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10 Volume Unit Referred
261      * Ext Used Recfm Lrecl BlkSz Dsorg Dsname B10142 3390 2006/03/20 2 31 F 80
262      * 80 PS MDI.OKL.WORK ARCIVE Not Direct Access Device
263      * KJ.IOP998.ERROR.PL.UNITTEST B1N231 3390 2006/03/20 1 15 VB 256 27998 PO
264      * PLU B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB
265      * 
266      * ----------------------------------- Group within Regex [1] Volume [2]
267      * Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record
268      * format [7] Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg:
269      * Dataset organisation. Many exists but only support: PS, PO, PO-E [10]
270      * Dsname: Dataset name
271      * 
272      * Note: When volume is ARCIVE, it means the dataset is stored somewhere in
273      * a tape archive. These entries is currently not supported by this parser.
274      * A null value is returned.
275      * 
276      * @param file
277      *            will be updated with Name, Type, Timestamp if parsed.
278      * @param entry zosDirectoryEntry
279      * @return true: entry was parsed, false: entry was not parsed.
280      */
281     private boolean parseFileList(FTPFile file, String entry) {
282         if (matches(entry)) {
283             file.setRawListing(entry);
284             String name = group(2);
285             String dsorg = group(1);
286             file.setName(name);
287 
288             // DSORG
289             if ("PS".equals(dsorg)) {
290                 file.setType(FTPFile.FILE_TYPE);
291             } 
292             else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) {
293                 // regex already ruled out anything other than PO or PO-E
294                 file.setType(FTPFile.DIRECTORY_TYPE);
295             } 
296             else {
297                 return false;
298             }
299             
300             return true;
301         }
302         
303         return false;
304     }
305 
306     /**
307      * Parse entries within a partitioned dataset.
308      * 
309      * Format of a memberlist within a PDS: 1 2 3 4 5 6 7 8 9 Name VV.MM Created
310      * Changed Size Init Mod Id TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11
311      * 0 KIL001 TBTOOL 01.12 2002/09/12 2004/11/26 19:54 51 28 0 KIL001
312      * 
313      * ------------------------------------------- [1] Name [2] VV.MM: Version .
314      * modification [3] Created: yyyy / MM / dd [4,5] Changed: yyyy / MM / dd
315      * HH:mm [6] Size: number of lines [7] Init: number of lines when first
316      * created [8] Mod: number of modified lines a last save [9] Id: User id for
317      * last update
318      * 
319      * 
320      * @param file
321      *            will be updated with Name, Type and Timestamp if parsed.
322      * @param entry zosDirectoryEntry
323      * @return true: entry was parsed, false: entry was not parsed.
324      */
325     private boolean parseMemberList(FTPFile file, String entry) {
326         if (matches(entry)) {
327             file.setRawListing(entry);
328             String name = group(1);
329             String datestr = group(2) + " " + group(3);
330             file.setName(name);
331             file.setType(FTPFile.FILE_TYPE);
332             try {
333                 file.setTimestamp(super.parseTimestamp(datestr));
334             } catch (ParseException e) {
335                 e.printStackTrace();
336                 // just ignore parsing errors.
337                 // TODO check this is ok
338                 return false; // this is a parsing failure too.
339             }
340             return true;
341         }
342 
343         return false;
344     }
345 
346     /**
347      * Assigns the name to the first word of the entry. Only to be used from a
348      * safe context, for example from a memberlist, where the regex for some
349      * reason fails. Then just assign the name field of FTPFile.
350      * 
351      * @param file
352      * @param entry
353      * @return
354      */
355     private boolean parseSimpleEntry(FTPFile file, String entry) {
356         if (entry != null && entry.length() > 0) {
357             file.setRawListing(entry);
358             String name = entry.split(" ")[0];
359             file.setName(name);
360             file.setType(FTPFile.FILE_TYPE);
361             return true;
362         }
363         return false;
364     }
365 
366     /**
367      * Parse the entry as a standard unix file. Using the UnixFTPEntryParser.
368      * 
369      * @param file
370      * @param entry
371      * @return true: entry is parsed, false: entry could not be parsed.
372      */
373     private boolean parseUnixList(FTPFile file, String entry) {
374         file = unixFTPEntryParser.parseFTPEntry(entry);
375         if (file == null)
376             return false;
377         return true;
378     }
379 
380     /**
381      * Matches these entries, note: no header: [1] [2] [3] [4] [5] IBMUSER1
382      * JOB01906 OUTPUT 3 Spool Files
383      * 012345678901234567890123456789012345678901234 1 2 3 4
384      * ------------------------------------------- Group in regex [1] Job name
385      * [2] Job number [3] Job status (INPUT,ACTIVE,OUTPUT) [4] Number of sysout
386      * files [5] The string "Spool Files"
387      * 
388      * 
389      * @param file
390      *            will be updated with Name, Type and Timestamp if parsed.
391      * @param entry zosDirectoryEntry
392      * @return true: entry was parsed, false: entry was not parsed.
393      */
394     private boolean parseJeslevel1List(FTPFile file, String entry) {
395         if (matches(entry)) {
396             if (group(3).equalsIgnoreCase("OUTPUT")) {
397                 file.setRawListing(entry);
398                 String name = group(2); /* Job Number, used by GET */
399                 file.setName(name);
400                 file.setType(FTPFile.FILE_TYPE);
401                 return true;
402             }
403         }
404 
405         return false;
406     }
407 
408     /**
409      * Matches these entries, note: no header: [1] [2] [3] [4] [5] JOBNAME JOBID
410      * OWNER STATUS CLASS IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool
411      * files IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files
412      * 012345678901234567890123456789012345678901234 1 2 3 4
413      * ------------------------------------------- Group in regex [1] Job name
414      * [2] Job number [3] Owner [4] Job status (INPUT,ACTIVE,OUTPUT) [5] Job
415      * Class [6] The rest
416      * 
417      * 
418      * @param file
419      *            will be updated with Name, Type and Timestamp if parsed.
420      * @param entry zosDirectoryEntry
421      * @return true: entry was parsed, false: entry was not parsed.
422      */
423     private boolean parseJeslevel2List(FTPFile file, String entry) {
424         if (matches(entry)) {
425             if (group(4).equalsIgnoreCase("OUTPUT")) {
426                 file.setRawListing(entry);
427                 String name = group(2); /* Job Number, used by GET */
428                 file.setName(name);
429                 file.setType(FTPFile.FILE_TYPE);
430                 return true;
431             }
432         }
433 
434         return false;
435     }
436 
437     /**
438      * preParse is called as part of the interface. Per definition is is called
439      * before the parsing takes place. Three kind of lists is recognize:
440      * z/OS-MVS File lists z/OS-MVS Member lists unix file lists
441      * @since 2.0
442      */
443     @Override
444     public List<String> preParse(List<String> orig) {
445         // simply remove the header line. Composite logic will take care of the
446         // two different types of
447         // list in short order.
448         if (orig != null && orig.size() > 0) {
449             String header = orig.get(0);
450             if (header.indexOf("Volume") >= 0 && header.indexOf("Dsname") >= 0) {
451                 setType(FILE_LIST_TYPE);
452                 super.setRegex(FILE_LIST_REGEX);
453             } else if (header.indexOf("Name") >= 0 && header.indexOf("Id") >= 0) {
454                 setType(MEMBER_LIST_TYPE);
455                 super.setRegex(MEMBER_LIST_REGEX);
456             } else if (header.indexOf("total") == 0) {
457                 setType(UNIX_LIST_TYPE);
458                 unixFTPEntryParser = new UnixFTPEntryParser();
459             } else if (header.indexOf("Spool Files") >= 30) {
460                 setType(JES_LEVEL_1_LIST_TYPE);
461                 super.setRegex(JES_LEVEL_1_LIST_REGEX);
462             } else if (header.indexOf("JOBNAME") == 0
463                     && header.indexOf("JOBID") > 8) {// header contains JOBNAME JOBID OWNER // STATUS CLASS
464                 setType(JES_LEVEL_2_LIST_TYPE);
465                 super.setRegex(JES_LEVEL_2_LIST_REGEX);
466             } else {
467                 setType(UNKNOWN_LIST_TYPE);
468             }
469             
470             if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary
471                 orig.remove(0);
472             }
473         }
474         
475         return orig;
476     }
477     
478     /**
479      * Explicitly set the type of listing being processed.
480      * @param type The listing type.
481      */
482     void setType(int type) {
483         isType = type;
484     }
485 
486     /* 
487      * @return
488      */
489     @Override
490     protected FTPClientConfig getDefaultConfiguration() {
491         return new FTPClientConfig(FTPClientConfig.SYST_MVS,
492                 DEFAULT_DATE_FORMAT, null, null, null, null);
493     }
494 
495 }