testrunner-lite: testdefinitionprocessor.c Source File

00001 /*
00002  * This file is part of testrunner-lite
00003  *
00004  * Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).
00005  *
00006  * Contact: Sampo Saaristo <ext-sampo.2.saaristo@nokia.com>
00007  *
00008  * This program is free software; you can redistribute it and/or
00009  * modify it under the terms of the GNU Lesser General Public License
00010  * version 2.1 as published by the Free Software Foundation.
00011  *
00012  * This program is distributed in the hope that it will be useful, but
00013  * WITHOUT ANY WARRANTY; without even the implied warranty of
00014  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
00015  * Lesser General Public License for more details.
00016  *
00017  * You should have received a copy of the GNU Lesser General Public
00018  * License along with this program; if not, write to the Free Software
00019  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
00020  * 02110-1301 USA
00021  *
00022  */
00023 
00024 /* ------------------------------------------------------------------------- */
00025 /* INCLUDE FILES */
00026 #include <stdio.h>
00027 #include <stdlib.h>
00028 #include <getopt.h>
00029 #include <errno.h>
00030 #include <string.h>
00031 #include <fcntl.h>
00032 #include <unistd.h>
00033 #include <sys/types.h>
00034 #include <sys/stat.h>
00035 #include <libxml/parser.h>
00036 #include <libxml/tree.h>
00037 #include <signal.h>
00038 #include <ctype.h>
00039 
00040 #include "testrunnerlite.h"
00041 #include "testdefinitionparser.h"
00042 #include "testresultlogger.h"
00043 #include "testfilters.h"
00044 #include "executor.h"
00045 #include "remote_executor.h"
00046 #include "manual_executor.h"
00047 #include "utils.h"
00048 #include "log.h"
00049 
00050 /* ------------------------------------------------------------------------- */
00051 /* EXTERNAL DATA STRUCTURES */
00052 /* None */
00053 
00054 /* ------------------------------------------------------------------------- */
00055 /* EXTERNAL GLOBAL VARIABLES */
00056 extern char* optarg;
00057 
00058 /* ------------------------------------------------------------------------- */
00059 /* EXTERNAL FUNCTION PROTOTYPES */
00060 /* None */
00061 
00062 /* ------------------------------------------------------------------------- */
00063 /* GLOBAL VARIABLES */
00064 struct timeval created;
00065 testrunner_lite_options opts;
00066 char *global_failure = NULL;
00067 int bail_out = 0;
00068 /* ------------------------------------------------------------------------- */
00069 /* CONSTANTS */
00070 /* None */
00071 
00072 /* ------------------------------------------------------------------------- */
00073 /* MACROS */
00074 /* None */
00075 
00076 /* ------------------------------------------------------------------------- */
00077 /* LOCAL GLOBAL VARIABLES */
00078 LOCAL td_suite *current_suite = NULL; /* Suite currently executed */
00079 LOCAL td_set   *current_set = NULL;   /* Set currently executed */
00080 LOCAL xmlChar  *cur_case_name = "";   /* Name of the current case or pre/post */
00081 LOCAL int       cur_step_num;         /* Number of current step within case */
00082 
00083 LOCAL int passcount = 0;
00084 LOCAL int casecount = 0;
00085 /* ------------------------------------------------------------------------- */
00086 /* LOCAL CONSTANTS AND MACROS */
00087 /* None */
00088 
00089 /* ------------------------------------------------------------------------- */
00090 /* MODULE DATA STRUCTURES */
00091 /* None */
00092 
00093 /* ------------------------------------------------------------------------- */
00094 /* LOCAL FUNCTION PROTOTYPES */
00095 /* ------------------------------------------------------------------------- */
00096 LOCAL void process_suite(td_suite *);
00097 /* ------------------------------------------------------------------------- */
00098 LOCAL void process_set(td_set *);
00099 /* ------------------------------------------------------------------------- */
00100 LOCAL int process_case (const void *, const void *);
00101 /* ------------------------------------------------------------------------- */
00102 LOCAL int case_result_na (const void *, const void *);
00103 /* ------------------------------------------------------------------------- */
00104 LOCAL int process_get (const void *, const void *);
00105 /* ------------------------------------------------------------------------- */
00106 LOCAL int step_execute (const void *, const void *);
00107 /* ------------------------------------------------------------------------- */
00108 LOCAL int prepost_steps_execute (const void *, const void *);
00109 /* ------------------------------------------------------------------------- */
00110 LOCAL int step_result_na (const void *, const void *);
00111 /* ------------------------------------------------------------------------- */
00112 LOCAL int step_post_process (const void *, const void *);
00113 /* ------------------------------------------------------------------------- */
00114 /* FORWARD DECLARATIONS */
00115 /* None */
00116 
00117 /* ------------------------------------------------------------------------- */
00118 /* ==================== LOCAL FUNCTIONS ==================================== */
00119 /* ------------------------------------------------------------------------- */
00125 LOCAL int step_execute (const void *data, const void *user) 
00126 {
00127         int res = CASE_PASS;
00128         td_step *step = (td_step *)data;
00129         td_case *c = (td_case *)user;
00130         exec_data edata;
00131 
00132         cur_step_num++;
00133 
00134         memset (&edata, 0x0, sizeof (exec_data));
00135         if (bail_out) {
00136                 res = CASE_FAIL;
00137                 step->has_result = 1; 
00138                 step->return_code = bail_out;
00139                 if (global_failure) {
00140                         step->failure_info = xmlCharStrdup (global_failure);
00141                         c->failure_info = xmlCharStrdup (global_failure);
00142                 }
00143                 goto out;
00144         }
00145         
00146         if (step->manual) {
00147                 if (!c->gen.manual)
00148                         LOG_MSG (LOG_WARNING, "Executing manual step from "
00149                                  "automatic case %s "
00150                                  "(generally not a good idea)",
00151                                  c->gen.name);
00152                 res = execute_manual (step);
00153                 goto out;
00154         }
00155         
00156         init_exec_data(&edata);
00157         
00158         if (c->dummy) {
00159                 /* Pre or post step */
00160                 edata.redirect_output = DONT_REDIRECT_OUTPUT;
00161         } else {
00162                 edata.redirect_output = REDIRECT_OUTPUT;
00163         }
00164         edata.soft_timeout = c->gen.timeout;
00165         edata.hard_timeout = COMMON_HARD_TIMEOUT;
00166         
00167         if (step->step) {
00168                 execute((char*)step->step, &edata);
00169                 
00170                 if (step->stdout_) free (step->stdout_);
00171                 if (step->stderr_) free (step->stderr_);
00172                 if (step->failure_info) free (step->failure_info);
00173                 
00174                 if (edata.stdout_data.buffer) {
00175                         step->stdout_ = edata.stdout_data.buffer;
00176                 }
00177                 if (edata.stderr_data.buffer) {
00178                         step->stderr_ = edata.stderr_data.buffer;
00179                 }
00180                 if (edata.failure_info.buffer) {
00181                         step->failure_info = edata.failure_info.buffer;
00182                         c->failure_info = xmlCharStrdup ((char *)
00183                                                          step->failure_info);
00184                         
00185                         LOG_MSG (LOG_INFO, "FAILURE INFO: %s",
00186                                  step->failure_info);
00187                 }
00188                 
00189                 step->pgid = edata.pgid; 
00190                 step->pid = edata.pid;
00191                 step->has_result = 1;
00192                 step->return_code = edata.result;
00193                 step->start = edata.start_time;
00194                 step->end = edata.end_time;
00195                 /*
00196                 ** Post and pre steps fail only if the expected result is 
00197                 *  specified
00198                 */
00199                 if (c->dummy) {
00200                         if (step->has_expected_result &&
00201                             (step->return_code != step->expected_result)) {
00202                                 LOG_MSG (LOG_INFO, 
00203                                          "STEP: %s return %d expected %d\n",
00204                                          step->step, step->return_code, 
00205                                          step->expected_result);
00206                                 res = CASE_FAIL;
00207                         }
00208                 } else if (step->return_code != step->expected_result) {
00209                         LOG_MSG (LOG_INFO, "STEP: %s return %d expected %d\n",
00210                                  step->step, step->return_code, 
00211                                  step->expected_result);
00212                         res = CASE_FAIL;
00213                 }
00214         }
00215  out:
00216         if (res != CASE_PASS)
00217                 c->case_res = res;
00218         
00219         
00220         return (res == CASE_PASS);
00221 }
00222 
00223 /* ------------------------------------------------------------------------- */
00229 LOCAL int prepost_steps_execute (const void *data, const void *user)
00230 {
00231         td_steps *steps = (td_steps *)data;
00232         td_case *dummy = (td_case *)user;
00233         
00234         if (steps->timeout == 0) {
00235                 dummy->gen.timeout = 180; /* default for pre/post steps */
00236         } else {
00237                 dummy->gen.timeout = steps->timeout;
00238         }
00239         
00240         if (xmlListSize(steps->steps) > 0) {
00241                 xmlListWalk (steps->steps, step_execute, dummy);
00242         }
00243         
00244         return 1;
00245 }
00246 
00247 /* ------------------------------------------------------------------------- */
00253 LOCAL int step_result_na (const void *data, const void *user) 
00254 {
00255         td_step *step = (td_step *)data;
00256         char *failure_info = (char *)user;
00257 
00258         step->has_result = 1;
00259         step->return_code = step->expected_result + 255; 
00260         step->failure_info = xmlCharStrdup (failure_info);
00261         
00262         return 1;
00263 }
00264 
00265 /* ------------------------------------------------------------------------- */
00272 LOCAL int step_post_process (const void *data, const void *user) 
00273 {
00274         td_step *step = (td_step *)data;
00275         td_case *c = (td_case *)user;
00276 
00277         /* No post processing for manual steps ... */
00278         if (step->manual) 
00279                 goto out;
00280         /* ... or filtered ones ... */
00281         if (c->filtered)
00282                 goto out;
00283 
00284         /* ... or ones that are not run ... */
00285         if (!step->start)
00286                 goto out;
00287 
00288         /* ... or ones that do not have process group ... */
00289         if (!step->pgid)
00290                 goto out;
00291 
00292         if (opts.target_address) {
00293                 ssh_kill (opts.target_address, step->pid);
00294         } 
00295         kill_pgroup(step->pgid, SIGKILL);
00296         
00297  out:
00298         return 1;
00299 }
00300 
00301 /* ------------------------------------------------------------------------- */
00307 LOCAL int process_case (const void *data, const void *user) 
00308 {
00309 
00310         td_case *c = (td_case *)data;
00311         
00312         if (c->gen.manual && !opts.run_manual) {
00313                 LOG_MSG(LOG_DEBUG, "Skipping manual case %s",
00314                         c->gen.name);
00315                 c->filtered = 1;
00316                 return 1;
00317         }
00318         if (!c->gen.manual && !opts.run_automatic) {
00319                 LOG_MSG(LOG_DEBUG, "Skipping automatic case %s",
00320                         c->gen.name);
00321                 c->filtered = 1;
00322                 return 1;
00323         }
00324         if (filter_case (c)) {
00325                 LOG_MSG (LOG_INFO, "Test case %s is filtered", c->gen.name);
00326                 return 1;
00327         }
00328         
00329         cur_case_name = c->gen.name;
00330         LOG_MSG (LOG_INFO, "Starting test case %s", c->gen.name);
00331         casecount++;
00332         
00333         c->case_res = CASE_PASS;
00334         if (c->gen.timeout == 0)
00335                 c->gen.timeout = COMMON_SOFT_TIMEOUT; /* the default one */
00336         
00337         if (c->gen.manual && opts.run_manual)
00338                 pre_manual (c);
00339         cur_step_num = 0;
00340         if (xmlListSize (c->steps) == 0) {
00341                 LOG_MSG (LOG_WARNING, "Case with no steps (%s).",
00342                          c->gen.name);
00343         }
00344 
00345         xmlListWalk (c->steps, step_execute, data);
00346         xmlListWalk (c->steps, step_post_process, data);
00347         
00348         if (c->gen.manual && opts.run_manual)
00349                 post_manual (c);
00350         
00351         LOG_MSG (LOG_INFO, "Finished test case Result: %s", 
00352                  case_result_str(c->case_res));
00353         passcount += (c->case_res == CASE_PASS);
00354         
00355         return 1;
00356 }
00357 /* ------------------------------------------------------------------------- */
00363 LOCAL int case_result_na (const void *data, const void *user)
00364 {
00365 
00366         td_case *c = (td_case *)data;
00367         char *failure_info = (char *)user;
00368 
00369         LOG_MSG (LOG_DEBUG, "Setting FAIL result for case %s", c->gen.name);
00370 
00371         c->case_res = CASE_FAIL;
00372         c->failure_info = xmlCharStrdup (failure_info);
00373 
00374         xmlListWalk (c->steps, step_result_na, user);
00375         
00376         return 1;
00377 }
00378 
00379 /* ------------------------------------------------------------------------- */
00385 LOCAL int process_get (const void *data, const void *user)
00386 {
00387 
00388         td_file *file = (td_file *)data;
00389         xmlChar *command;
00390         char *fname;
00391         exec_data edata;
00392         char    *remote = opts.target_address;
00393 
00394         memset (&edata, 0x0, sizeof (exec_data));
00395         init_exec_data(&edata);
00396         edata.soft_timeout = COMMON_SOFT_TIMEOUT;
00397         edata.hard_timeout = COMMON_HARD_TIMEOUT;
00398 
00399         fname = malloc (strlen((char *)file->filename) + 1);
00400         trim_string ((char *)file->filename, fname);
00401 
00402         /*
00403         ** Compose command 
00404         */
00405         if (remote) {
00406                 opts.target_address = NULL; /* execute locally */
00407                 command = (xmlChar *)malloc (strlen ("scp ") + 
00408                                              strlen (fname) +
00409                                              strlen (opts.output_folder) +
00410                                              strlen (remote) + 10);
00411                 sprintf ((char *)command, "scp %s:\'%s\' %s", remote, fname,
00412                          opts.output_folder);
00413 
00414         } else {
00415                 command = (xmlChar *)malloc (strlen ("cp ") + 
00416                                              strlen (fname) +
00417                                              strlen (opts.output_folder) + 2);
00418                 sprintf ((char *)command, "cp %s %s", fname,
00419                          opts.output_folder);
00420         }
00421         LOG_MSG (LOG_DEBUG, "%s:  Executing command: %s", PROGNAME, 
00422                  (char*)command);
00423         /*
00424         ** Execute it
00425         */
00426         execute((char*)command, &edata);
00427 
00428         if (edata.result) {
00429                 LOG_MSG (LOG_ERR, "%s: %s failed: %s\n", PROGNAME, command,
00430                          (char *)(edata.stderr_data.buffer ?
00431                                   edata.stderr_data.buffer : 
00432                                   BAD_CAST "no info available"));
00433         }
00434         opts.target_address = remote;
00435         if (edata.stdout_data.buffer) free (edata.stdout_data.buffer);
00436         if (edata.stderr_data.buffer) free (edata.stderr_data.buffer);
00437         if (edata.failure_info.buffer) free (edata.failure_info.buffer);
00438 
00439         if (!file->delete_after)
00440                 goto out;
00441 
00442         memset (&edata, 0x0, sizeof (exec_data));
00443         init_exec_data(&edata);
00444         edata.soft_timeout = COMMON_SOFT_TIMEOUT;
00445         edata.hard_timeout = COMMON_HARD_TIMEOUT;
00446         sprintf ((char *)command, "rm -f %s", fname);
00447         LOG_MSG (LOG_DEBUG, "%s:  Executing command: %s", PROGNAME, 
00448                  (char*)command);
00449         execute((char*)command, &edata);
00450         if (edata.result) {
00451                 LOG_MSG (LOG_ERR, "%s: %s failed: %s\n", PROGNAME, command,
00452                          (char *)(edata.stderr_data.buffer ?
00453                                   edata.stderr_data.buffer : 
00454                                   BAD_CAST "no info available"));
00455         }
00456         if (edata.stdout_data.buffer) free (edata.stdout_data.buffer);
00457         if (edata.stderr_data.buffer) free (edata.stderr_data.buffer);
00458         if (edata.failure_info.buffer) free (edata.failure_info.buffer);
00459 
00460  out:
00461         free (command);
00462         free (fname);
00463         return 1;
00464 }
00465 /* ------------------------------------------------------------------------- */
00469 LOCAL void process_suite (td_suite *s)
00470 {
00471         LOG_MSG (LOG_INFO, "Test suite: %s", s->gen.name);
00472 
00473         write_pre_suite_tag (s);
00474         current_suite = s;
00475         
00476 }
00477 /* ------------------------------------------------------------------------- */
00480 LOCAL void end_suite ()
00481 {
00482         write_post_suite_tag ();
00483         td_suite_delete (current_suite);
00484         current_suite = NULL;
00485 }
00486 /* ------------------------------------------------------------------------- */
00490 LOCAL void process_set (td_set *s)
00491 {
00492         td_case dummy;
00493 
00494         /*
00495         ** Check that the set is not filtered
00496         */
00497         if (filter_set (s)) {
00498                 LOG_MSG (LOG_INFO, "Test set %s is filtered", s->gen.name);
00499                 goto skip_all;
00500         }
00501 
00502         /*
00503         ** Check that the set is supposed to be executed in the current env
00504         */
00505         s->environment = xmlCharStrdup (opts.environment);
00506         if (!xmlListSearch (s->environments, opts.environment)) {
00507                 LOG_MSG (LOG_INFO, "Test set %s not run on "
00508                          "environment: %s", 
00509                          s->gen.name, opts.environment);
00510                 goto skip_all;
00511         }
00512         current_set = s;
00513         LOG_MSG (LOG_INFO, "Test set: %s", s->gen.name);
00514         write_pre_set_tag (s);
00515 
00516         if (xmlListSize (s->pre_steps) > 0) {
00517                 cur_case_name = (xmlChar *)"pre_steps";
00518                 cur_step_num = 0;
00519                 memset (&dummy, 0x0, sizeof (td_case));
00520                 dummy.case_res = CASE_PASS;
00521                 dummy.dummy = 1;
00522                 LOG_MSG (LOG_INFO, "Executing pre steps");
00523                 xmlListWalk (s->pre_steps, prepost_steps_execute, &dummy);
00524                 if (dummy.case_res != CASE_PASS) {
00525                         LOG_MSG (LOG_ERR, "Pre steps failed. "
00526                                  "Test set %s aborted.", s->gen.name); 
00527                         xmlListWalk (s->cases, case_result_na, 
00528                                      global_failure ? global_failure :
00529                                      "pre_steps failed");
00530                         goto short_circuit;
00531                 }
00532         }
00533         
00534         xmlListWalk (s->cases, process_case, s);
00535         if (xmlListSize (s->post_steps) > 0) {
00536                 LOG_MSG (LOG_INFO, "Executing post steps");
00537                 cur_case_name = (xmlChar *)"post_steps";
00538                 cur_step_num = 0;
00539                 memset (&dummy, 0x0, sizeof (td_case));
00540                 dummy.case_res = CASE_PASS;
00541                 dummy.dummy = 1;
00542                 xmlListWalk (s->post_steps, prepost_steps_execute, &dummy);
00543                 if (dummy.case_res == CASE_FAIL)
00544                         LOG_MSG (LOG_ERR, 
00545                                  "Post steps failed for %s.", s->gen.name);
00546         }
00547         xmlListWalk (s->gets, process_get, s);
00548 
00549  short_circuit:
00550         write_post_set_tag (s);
00551         if (xmlListSize (s->pre_steps) > 0)
00552                 xmlListWalk (s->pre_steps, step_post_process, &dummy);
00553         if (xmlListSize (s->post_steps) > 0)
00554                 xmlListWalk (s->post_steps, step_post_process, &dummy);
00555         xml_end_element();
00556  skip_all:
00557         td_set_delete (s);
00558         return;
00559 }
00560 /* ------------------------------------------------------------------------- */
00561 /* ======================== FUNCTIONS ====================================== */
00562 /* ------------------------------------------------------------------------- */
00566 void td_process () {
00567         int retval;
00568         td_parser_callbacks cbs;
00569 
00570         memset (&cbs, 0x0, sizeof(td_parser_callbacks));
00571         /*
00572         ** Set callbacks for parser
00573         */
00574         cbs.test_suite = process_suite;
00575         cbs.test_suite_end = end_suite;
00576         cbs.test_set = process_set;
00577 
00578         retval = td_register_callbacks (&cbs);
00579         
00580         /*
00581         ** Call td_next_node untill error occurs or the end of data is reached
00582         */
00583         LOG_MSG (LOG_INFO, "Starting to run tests...");
00584 
00585         while (td_next_node() == 0);
00586 
00587         LOG_MSG (LOG_INFO, "Finished running tests.");
00588         LOG_MSG (LOG_INFO, "Executed %d cases. Passed %d Failed %d",
00589                  casecount, passcount, casecount - passcount);
00590         return; 
00591 }       
00592 /* ------------------------------------------------------------------------- */
00596 const char *current_set_name ()
00597 {
00598         if (current_set)
00599                 return (char *)current_set->gen.name;
00600         return "";
00601 }
00602 /* ------------------------------------------------------------------------- */
00606 const char *current_case_name ()
00607 {
00608         return (char *)cur_case_name;
00609 }
00610 /* ------------------------------------------------------------------------- */
00614 int current_step_num ()
00615 {
00616         return cur_step_num;
00617 }
00618 
00619 /* ================= OTHER EXPORTED FUNCTIONS ============================== */
00620 /* None */
00621 
00622 /* ------------------------------------------------------------------------- */
00623 /* End of file */