file_utils.inc

Go to the documentation of this file.

<?php

function tpps_parse_file($fid, $max_rows = 0, $no_header = FALSE, $columns = NULL) {
  $content = array();
  $options = array(
    'no_header' => $no_header,
    'columns' => $columns,
    'max_rows' => $max_rows,
    'content' => &$content,
  );

  tpps_file_iterator($fid, 'tpps_parse_file_helper', $options);
  $content['headers'] = tpps_file_headers($fid, $no_header);
  return $content;
}

function tpps_parse_file_helper($row, array &$options) {
  $options['content'][] = $row;
}

function tpps_file_len($fid) {
  $file = file_load($fid);
  $location = tpps_get_location($file->uri);
  $extension = tpps_get_path_extension($location);
  $count = 0;
  $options = array(
    'count' => &$count,
  );
  if ($extension == 'vcf') {
    $options['skip_prefix'] = '#';
  }
  tpps_file_iterator($fid, 'tpps_file_len_helper', $options);
  return $count;
}

function tpps_file_len_helper($row, array &$options = array()) {
  if (empty($options['skip_prefix'])) {
    $options['count']++;
  }
  else {
    if (substr(current($row), 0, strlen($options['skip_prefix'])) != $options['skip_prefix']) {
      $options['count']++;
    }
  }
}

function tpps_file_width($fid) {
  $file = file_load($fid);
  $location = tpps_get_location($file->uri);
  $extension = tpps_get_path_extension($location);
  switch ($extension) {
    case 'xlsx':
      return tpps_xlsx_width($location);

    case 'txt':
    case 'csv':
      return tpps_flat_width($location);

    case 'vcf':
    case 'gz':
      return tpps_flat_width($location, "\t");

    default:
      throw new Exception('Unrecognized file type');
  }
}

function tpps_xlsx_width($location) {
  $dir = drupal_realpath(TPPS_TEMP_XLSX);

  $zip = new ZipArchive();
  $zip->open($location);
  $zip->extractTo($dir);

  $data_location = $dir . '/xl/worksheets/sheet1.xml';

  $dimension = tpps_xlsx_get_dimension($data_location);
  preg_match('/([A-Z]+)[0-9]+:([A-Z]+)[0-9]+/', $dimension, $matches);
  $left_hex = unpack('H*', $matches[1]);
  $hex = $left_hex[1];
  $right_hex = unpack('H*', $matches[2]);

  $len = 0;
  while (base_convert($hex, 16, 10) <= base_convert($right_hex[1], 16, 10)) {
    $len++;
    $hex = tpps_increment_hex($hex);
  }
  return $len;
}

function tpps_flat_width($location, $delim = ',') {
  $original = ini_get('auto_detect_line_endings');
  if (!$original) {
    ini_set('auto_detect_line_endings', TRUE);
  }

  $longest = 0;
  $handle = gzopen($location, 'r');

  while (($line = gzgets($handle))) {
    $line = str_getcsv($line, $delim);
    $longest = max($longest, count($line));
  }

  fclose($handle);
  ini_set('auto_detect_line_endings', $original);
  return $longest;
}

function tpps_increment_hex($hex) {
  $add = 1;
  $new = "";
  for ($i = strlen(pack('H*', $hex)) - 1; $i >= 0; $i--) {
    $num = base_convert(substr($hex, (2 * $i), 2), 16, 10) + $add;
    if ($num > 90) {
      $num = 65;
    }
    else {
      $add = 0;
    }
    $new = base_convert($num, 10, 16) . $new;
    if ($i == 0 and $add) {
      $new = '41' . $new;
    }
  }
  return $new;
}

function tpps_convert_colname($column) {
  $total = 0;
  $base_16 = unpack('H*', 'A')[1];
  $base = base_convert($base_16, 16, 10) - 1;
  for ($i = 0; $i < strlen($column); $i++) {
    $total *= 26;
    $num_16 = unpack('H*', $column[$i])[1];
    $total += base_convert($num_16, 16, 10) - $base;
  }
  return $total - 1;
}

function tpps_rename_file($fid, $new_name, array $options = array()) {
  if (!array_key_exists('preserve', $options)) {
    $options['preserve'] = FALSE;
  }

  if (!empty($fid) and ($file = file_load($fid))) {
    $new_filename = $new_name . "." . tpps_get_path_extension($file->uri);
    if (!preg_match('/^(.*\/)(.*)$/', $new_name, $matches)) {
      preg_match('/^(.*\/).*$/', $file->uri, $matches);
      $new_filename = $matches[1] . $new_filename;
    }

    if ($options['preserve']) {
      $file->status = FILE_STATUS_PERMANENT;
      file_save($file);
      // Check if file already exists then return that file.
      $result = db_query('SELECT f.fid
              FROM {file_managed} f WHERE f.uri = :uri', array(':uri' => $new_filename));
      $record = $result->fetchObject();
      if(isset($record)) {
        return file_load($record->fid);
      }
      return file_copy($file, $new_filename, FILE_EXISTS_RENAME);
    }
    // Check if file already exists then return that file.
    $result = db_query('SELECT f.fid
    FROM {file_managed} f WHERE f.uri = :uri', array(':uri' => $new_filename));
    $record = $result->fetchObject();

    if(isset($record->fid)) {
      // print_r("FID:" . $record->fid . "\n");
      return file_load($record->fid);
    }
    $file_object = file_move($file, $new_filename, FILE_EXISTS_RENAME);
    // print_r("FILE OBJECT:\n");
    // print_r($file_object);
    // print_r("\n");
    return $file_object;
  }
  throw new Exception("Error: could not open file");
}

function tpps_parse_file_column($fid, $column, $no_header = FALSE) {
  $content = array();
  $options = array(
    'no_header' => $no_header,
    'columns' => array($column),
    'content' => &$content,
  );
  tpps_file_iterator($fid, 'tpps_parse_file_column_helper', $options);
  return $content;
}

function tpps_parse_file_column_helper($row, array &$options) {
  $options['content'][] = $row[current($options['columns'])];
}

function tpps_compare_files($fid_1, $fid_2, $file_1_id_name, $file_2_id_name, $file_1_no_header = FALSE, $file_2_no_header = FALSE) {
  $missing = array();
  $file_1_content = array_unique(tpps_parse_file_column($fid_1, $file_1_id_name, $file_1_no_header));
  $file_2_content = array_unique(tpps_parse_file_column($fid_2, $file_2_id_name, $file_2_no_header));
  asort($file_1_content);
  asort($file_2_content);
  reset($file_1_content);
  reset($file_2_content);

  while (current($file_1_content) !== FALSE and current($file_2_content) !== FALSE) {
    if (current($file_1_content) < current($file_2_content)) {
      $missing[] = current($file_1_content);
      next($file_1_content);
      continue;
    }
    elseif (current($file_1_content) > current($file_2_content)) {
      next($file_2_content);
      continue;
    }
    next($file_1_content);
    next($file_2_content);
    continue;
  }

  while (current($file_1_content) !== FALSE) {
    $missing[] = current($file_1_content);
    next($file_1_content);
  }
  return $missing;
}

function tpps_file_validate_columns(array &$form_state, array $required_groups, array $file_element) {
  $cols = $file_element['#value']['columns'];

  $parents = $file_element['#parents'];
  $new_end_columns = end($parents) . "-columns";
  $new_end_no_header = end($parents) . "-no-header";
  $new_end_empty = end($parents) . "-empty";
  $group_path = array_pop($parents) . "-groups";
  $values = drupal_array_get_nested_value($form_state['values'], $parents);
  // Initialize form column values in form state.
  $values[$new_end_columns] = array();
  // Hold onto the location of the columns in form state.
  $state_column_values = &$values[$new_end_columns];
  $values[$new_end_no_header] = isset($file_element['#value']['no-header']) ? $file_element['#value']['no-header'] : NULL;
  $values[$new_end_empty] = isset($file_element['#value']['empty']) ? $file_element['#value']['empty'] : NULL;

  $title_parts = explode(':', $file_element['#title']);
  $error_prompt = $title_parts[0];

  $groups = array();
  $required_groups_flat = array();
  foreach ($required_groups as $group => $combinations) {
    $groups[$group] = array();
    $required_groups_flat[$group] = array();
    foreach ($combinations as $name => $combination) {
      $required_groups_flat[$group] = array_merge($required_groups_flat[$group], $combination);
    }
  }

  foreach ($cols as $name => $type) {
    $state_column_values[$name] = $type;
    foreach ($required_groups_flat as $group => $types) {
      if (in_array($type, $types)) {
        if (!isset($groups[$group][$type])) {
          $groups[$group][$type] = array($name);
        }
        else {
          $groups[$group][$type][] = $name;
        }
        break;
      }
    }
  }
  // dpm($required_groups);
  foreach ($required_groups as $group => $combinations) {
    $group_valid = FALSE;
    $groups[$group]['#type'] = array();
    foreach ($combinations as $name => $combination) {
      $combination_valid = TRUE;
      foreach ($combination as $type) {
        if (!isset($groups[$group][$type])) {
          $combination_valid = FALSE;
          break;
        }
      }
      if ($combination_valid) {
        $groups[$group]['#type'][] = $name;
        $group_valid = TRUE;
      }
    }

    if (!$group_valid) {
      form_set_error($file_element['#name'] . "[columns][$group", "$error_prompt: Please specify a column or columns that hold $group.");
    }
  }

  foreach ($groups as $key => $group) {
    foreach ($group as $opt_num => $col_names) {
      if (count($col_names) == 1) {
        $groups[$key][$opt_num] = $col_names[0];
      }
    }
  }

  $values[$group_path] = $groups;
  drupal_array_set_nested_value($form_state['values'], $parents, $values);

  return $groups;
}

function tpps_save_file_columns(array &$form, array &$form_state) {

  if ($form_state['stage'] == TPPS_PAGE_3) {
    for ($i = 1; $i <= $form_state['stats']['species_count']; $i++) {
      if (($file = file_load($form_state['values']['tree-accession']["species-$i"]['file'])) and $form_state['values']['tree-accession']["species-$i"]['file'] != 0) {
        file_usage_add($file, 'tpps', 'tpps_project', substr($form_state['accession'], 4));
      }

      $form_state['values']['tree-accession']["species-$i"]['file-columns'] = array();

      if (isset($form['tree-accession']["species-$i"]['file']['#value']['columns'])) {
        foreach ($form['tree-accession']["species-$i"]['file']['#value']['columns'] as $col => $val) {
          if ($col[0] != '#') {
            $form_state['values']['tree-accession']["species-$i"]['file-columns'][$col] = $form['tree-accession']["species-$i"]['file']['#value']['columns'][$col];
          }
        }
      }

      if (empty($form_state['values']['tree-accession']['check'])) {
        break;
      }
    }
  }
  elseif ($form_state['stage'] == TPPS_PAGE_4) {
    if (isset($form['organism-1']['genotype'])) {
      for ($i = 1; $i <= $form_state['saved_values'][TPPS_PAGE_1]['organism']['number']; $i++) {

        if (!empty($form_state['values']["organism-$i"]['genotype']['files']['snps-assay']) and ($file = file_load($form_state['values']["organism-$i"]['genotype']['files']['snps-assay']))) {
          file_usage_add($file, 'tpps', 'tpps_project', substr($form_state['accession'], 4));

          $form_state['values']["organism-$i"]['genotype']['files']['snps-assay-columns'] = array();

          if (isset($form["organism-$i"]['genotype']['files']['snps-assay']['#value']['columns'])) {
            foreach ($form["organism-$i"]['genotype']['files']['snps-assay']['#value']['columns'] as $col => $val) {
              if ($col[0] != '#') {
                $form_state['values']["organism-$i"]['genotype']['files']['snps-assay-columns'][$col] = $form["organism-$i"]['genotype']['files']['snps-assay']['#value']['columns'][$col];
              }
            }
          }
        }

        if (!empty($form_state['values']["organism-$i"]['genotype']['files']['other']) and ($file = file_load($form_state['values']["organism-$i"]['genotype']['files']['other']))) {
          file_usage_add($file, 'tpps', 'tpps_project', substr($form_state['accession'], 4));

          $form_state['values']["organism-$i"]['genotype']['files']['other-columns'] = array();

          if (isset($form["organism-$i"]['genotype']['files']['other']['#value']['columns'])) {
            foreach ($form["organism-$i"]['genotype']['files']['other']['#value']['columns'] as $col => $val) {
              if ($col[0] != '#') {
                $form_state['values']["organism-$i"]['genotype']['files']['other-columns'][$col] = $form["organism-$i"]['genotype']['files']['other']['#value']['columns'][$col];
              }
            }
          }
        }
      }
    }

    if (isset($form['organism-1']['phenotype'])) {
      for ($i = 1; $i <= $form_state['saved_values'][TPPS_PAGE_1]['organism']['number']; $i++) {
        if (!empty($form_state['values']["organism-$i"]['phenotype']['normal-check'])) {
          if (($file = file_load($form_state['values']["organism-$i"]['phenotype']['file'])) and $form_state['values']["organism-$i"]['phenotype']['file'] != 0) {
            file_usage_add($file, 'tpps', 'tpps_project', substr($form_state['accession'], 4));
          }

          $form_state['values']["organism-$i"]['phenotype']['file-columns'] = array();

          if (isset($form["organism-$i"]['phenotype']['file']['#value']['columns'])) {
            foreach ($form["organism-$i"]['phenotype']['file']['#value']['columns'] as $col => $val) {
              if ($col[0] != '#') {
                $form_state['values']["organism-$i"]['phenotype']['file-columns'][$col] = $form["organism-$i"]['phenotype']['file']['#value']['columns'][$col];
              }
            }
          }

          if ($form_state['values']["organism-$i"]['phenotype']['check'] != '0') {
            if (($file = file_load($form_state['values']["organism-$i"]['phenotype']['metadata'])) and $form_state['values']["organism-$i"]['phenotype']['metadata'] != 0) {
              file_usage_add($file, 'tpps', 'tpps_project', substr($form_state['accession'], 4));
            }

            $form_state['values']["organism-$i"]['phenotype']['metadata-columns'] = array();

            if (isset($form["organism-$i"]['phenotype']['metadata']['#value']['columns'])) {
              foreach ($form["organism-$i"]['phenotype']['metadata']['#value']['columns'] as $col => $val) {
                if ($col[0] != '#') {
                  $form_state['values']["organism-$i"]['phenotype']['metadata-columns'][$col] = $form["organism-$i"]['phenotype']['metadata']['#value']['columns'][$col];
                }
              }
            }
          }
        }
      }
    }
  }
}

function tpps_get_location($location) {
  $location = drupal_realpath($location);
  $extension = tpps_get_path_extension($location);
  if ($extension == 'zip' or $extension == 'tar') {
    $files = tpps_get_archive_files($location);
    if ($files and (count($files) == 3 or (count($files) == 4 and substr($files[3], -9) === '/__MACOSX')) and !is_dir($files[2])) {
      $location = $files[2];
    }
  }
  return $location;
}

function tpps_get_path_extension($path) {
  preg_match('/\.([a-zA-Z0-9]*)$/', $path, $matches);

  if(count($matches) == 0) {
    return ""; // Added by Rish during debugging process. XLSX extracting creates files without paths
  }
  return strtolower($matches[1]);

}

function tpps_xlsx_get_dimension($location) {
  $reader = new XMLReader();
  $reader->open($location);
  while ($reader->read()) {
    if ($reader->nodeType == XMLReader::ELEMENT and $reader->name == 'dimension') {
      $dim = $reader->getAttribute('ref');
      $reader->close();
      return $dim;
    }
  }
  return NULL;
}

function tpps_xlsx_get_rows(array &$readers, array &$strings = array(), $plain = TRUE, $columns = NULL) {
  $rows = array();
  $empty = TRUE;
  foreach ($readers as $key => $r) {
    $reader = &$readers[$key];
    $row = tpps_xlsx_get_row($reader, $strings, $plain, $columns);
    if ($row) {
      $rows[$key] = $row;
      $empty = FALSE;
    }
    else {
      $rows[$key] = NULL;
    }
  }
  if ($empty) {
    return FALSE;
  }
  return $rows;
}

function tpps_xlsx_get_row(&$reader, array &$strings = array(), $plain = TRUE, $columns = NULL) {
  while ($reader->read()) {
    if ($reader->nodeType == XMLReader::ELEMENT and $reader->name == 'row') {
      if ($plain) {
        $cells = array();
        while (!($reader->nodeType == XMLReader::END_ELEMENT and $reader->name == 'row') and $reader->read()) {
          if ($reader->nodeType == XMLReader::ELEMENT and $reader->name == 'c') {
            $cell_type = $reader->getAttribute('t');
            preg_match('/([A-Z]+)([0-9]+)/', $reader->getAttribute('r'), $matches);
            $cell_location = $matches[1];
            if (!empty($columns) and !isset($columns[$cell_location])) {
              continue;
            }
            $cells[$cell_location] = tpps_xlsx_get_cell_value($reader, $cell_type, $strings);
          }
        }
        return $cells;
      }
      return TRUE;
    }
  }
  return FALSE;
}

function tpps_xlsx_translate_date($date) {
  if (strtotime($date) !== FALSE) {
    return $date;
  }

  if ($date > 60) {
    $date = $date - 1;
    return date("m/d/Y", strtotime("12/31/1899 +$date days"));
  }
  if ($date < 60) {
    return date("m/d/Y", strtotime("12/31/1899 +$date days"));
  }

  return NULL;
}

function tpps_xlsx_get_cell_value(&$reader, $cell_type, array &$strings = array()) {
  if ($reader->isEmptyElement) {
    return NULL;
  }

  while (!($reader->nodeType == XMLReader::END_ELEMENT and ($reader->name == 'c' or $reader->name == 'row')) and $reader->read()) {
    if ($reader->nodeType == XMLReader::ELEMENT and $reader->name == 'v') {
      $value = $reader->readInnerXml();
      if (!empty($cell_type) and $cell_type == 's') {
        $value = $strings[$value];
      }
      return $value;
    }
  }
  return NULL;
}

function tpps_xlsx_get_strings($strings_location) {
  $strings = array();
  $reader = new XMLReader();
  $reader->open($strings_location);
  $count = 0;
  while ($reader->read()) {
    if ($reader->nodeType == XMLReader::ELEMENT and $reader->name == 'si') {
      $string = "";

      while (!($reader->nodeType == XMLReader::END_ELEMENT and $reader->name == 'si') and $reader->read()) {
        if ($reader->nodeType == XMLReader::ELEMENT and $reader->name == 't') {
          $string .= $reader->readInnerXml();
        }
      }
      $strings[$count++] = $string;
    }
  }
  $reader->close();
  return $strings;
}

function tpps_rmdir($dir) {
  if (is_dir($dir)) {
    $children = scandir($dir);
    foreach ($children as $child) {
      if ($child != '.' and $child != '..') {
        if (is_dir($dir . '/' . $child) and !is_link($dir . '/' . $child)) {
          tpps_rmdir($dir . '/' . $child);
        }
        else {
          unlink($dir . '/' . $child);
        }
      }
    }
    rmdir($dir);
  }
}

function tpps_get_archive_files($archive) {
  $files = array();
  if (is_object($archive) and !empty($archive->uri)) {
    $archive = $archive->uri;
  }
  $loc = drupal_realpath($archive);

  $ext = tpps_get_path_extension($loc);
  $unzip_dir = drupal_realpath('temporary://tpps_tmp');
  if (is_dir($unzip_dir)) {
    tpps_rmdir($unzip_dir);
  }

  if (is_dir($unzip_dir) or mkdir($unzip_dir)) {
    switch ($ext) {
      case 'zip':
        $zip = new \ZipArchive();
        $zip->open($loc);
        $zip->extractTo($unzip_dir);
        break;

      case 'gz':
        $zip = new \Archive_Tar($loc, 'gz');
        $zip->extract($unzip_dir);
        break;

      case 'tar':
        $zip = new \Archive_Tar($loc);
        $zip->extract($unzip_dir);
        break;

      default:
        return array();
    }

    $dir = $unzip_dir;
    $files = scandir($dir);
    if ($files and count($files) == 3 and is_dir($dir . '/' . $files[2])) {
      $dir .= '/' . $files[2];
      $files = scandir($dir);
    }
  }

  foreach ($files as $key => $name) {
    $files[$key] = $dir . '/' . $name;
  }

  return $files;
}

function tpps_file_headers($fid, $no_header = FALSE) {
  $headers = array();
  if ($no_header) {
    $hex = unpack('H*', 'A')[1];
    $width = tpps_file_width($fid);
    for ($i = 0; $i < $width; $i++) {
      $key = pack('H*', $hex);
      $headers[$key] = $i;
      $hex = tpps_increment_hex($hex);
    }
    return $headers;
  }

  $content = array();
  $options = array(
    'no_header' => TRUE,
    'max_rows' => 1,
    'content' => &$content,
  );
  tpps_file_iterator($fid, 'tpps_parse_file_helper', $options);
  return current($content);
}

function tpps_file_iterator($fid, $function, array &$options = array()) {
  if (!function_exists($function)) {
    return FALSE;
  }

  if (!empty($options['job'])) {
    $options['job']->setTotalItems(tpps_file_len($fid));
    $options['job']->setItemsHandled(0);
  }

  $file = file_load($fid);
  $file_location = tpps_get_location($file->uri);

  foreach (tpps_file_generator($fid, $options) as $item) {
    // Assume the item IS NOT EMPTY
    $item_is_not_empty = true;
    if(is_array($item)) {  
      // Check each key value
      $values_count = count($item);
      $values_empty_count = 0;
      foreach($item as $key => $value) {
        if($value == '') {
          $values_empty_count = $values_empty_count + 1;
        }
      }
      // Check if $values_count == $values_empty_count (this means item is empty)
      if ($values_count == $values_empty_count) {
        $item_is_not_empty = false;
      }
    }

    // if the item is not empty, proceed to process the line
    if ($item_is_not_empty) {
      $function($item, $options);
      if (!empty($options['job'])) {
        $options['job']->addItemsHandled(1);
      }
    }
    else {
      // print_r($item);
      echo json_encode($item) . "\n";
      echo "[TPPS FILE ITERATOR] Found a line/item that was empty, ignoring this item\n";
      echo "[TPPS FILE ITERATOR] File location: " . $file_location . "\n";
      // throw new Exception('DEBUG STOP - FOUND AN EMPTY LINE');
    }
  }
  return TRUE;
}

function tpps_file_generator($fid, array $options = array()) {
  $file = file_load($fid);
  $location = tpps_get_location($file->uri);
  $extension = tpps_get_path_extension($location);
  switch ($extension) {
    case 'xlsx':
      return tpps_xlsx_generator($location, $options);

    case 'txt':
    case 'csv':
    case 'vcf':
    case 'gz':
      return tpps_flat_generator($location, $options);

    default:
      if (empty($extension)) {
        $extension = "NULL";
      }
      $msg = "Unrecognized file type: $extension";
      throw new Exception($msg);
  }
}

function tpps_xlsx_generator($location, array $options = array()) {
  $dir = drupal_realpath(TPPS_TEMP_XLSX);
  $no_header = $options['no_header'] ?? FALSE;
  $columns = $options['columns'] ?? NULL;
  $max_rows = $options['max_rows'] ?? NULL;

  if (!empty($columns)) {
    $new_columns = array();
    foreach ($columns as $col) {
      $new_columns[$col] = $col;
    }
    $columns = $new_columns;
  }

  $zip = new ZipArchive();
  $zip->open($location);
  $zip->extractTo($dir);

  $strings_location = $dir . '/xl/sharedStrings.xml';

  // Find all worksheets.
  $sheets = scandir($dir . '/xl/worksheets');
  unset($sheets[array_search('.', $sheets)]);
  unset($sheets[array_search('..', $sheets)]);

  $strings = tpps_xlsx_get_strings($strings_location);

  // Get dimensions, readers.
  $first_left = NULL;
  $last_right = NULL;
  $dims = array();
  $readers = array();
  foreach ($sheets as $sheet) {
    // dpm('SHEET location: ' . $dir . '/xl/worksheets/' . $sheet);
    if (tpps_get_path_extension($dir . '/xl/worksheets/' . $sheet) !== 'xml') {
      continue;
    }
    $loc = $dir . '/xl/worksheets/' . $sheet;
    $dimension = tpps_xlsx_get_dimension($loc);
    // dpm($loc);
    // dpm($dimension);
    preg_match('/([A-Z]+)[0-9]+:([A-Z]+)[0-9]+/', $dimension, $matches);
    // dpm($matches);
    if(count($matches) < 3) {
      // The dimensions are invalid (probably an empty worksheet)
    }
    else {
      // Matches found dimensions, continue processing this as a valid sheet
      $left_hex = unpack('H*', $matches[1]);
      $hex = $left_hex[1];
      $right_hex = unpack('H*', $matches[2]);
      if ($first_left == NULL) {
        $first_left = $left_hex[1];
        $last_right = $left_hex[1] - 1;
      }
      while (base_convert($hex, 16, 10) <= base_convert($right_hex[1], 16, 10)) {
        $hex = tpps_increment_hex($hex);
        $last_right = tpps_increment_hex($last_right);
      }
      $dims[] = array($left_hex[1], $right_hex[1]);
      $reader = new XMLReader();
      $reader->open($loc);
      $readers[] = $reader;
    }
  }

  // If the file has a header row, skip it.
  if (!$no_header) {
    tpps_xlsx_get_rows($readers, $strings);
  }

  // Iterate through file.
  $count = 0;
  while (($rows = tpps_xlsx_get_rows($readers, $strings, TRUE, $columns))) {
    if (!empty($max_rows) and $count >= $max_rows) {
      break;
    }
    $count++;

    $values = array();
    $key_hex = $first_left;
    foreach ($readers as $idx => $read) {
      $row = $rows[$idx];
      if (!empty($row)) {
        ksort($row);
      }
      $hex = $dims[$idx][0];
      while (base_convert($hex, 16, 10) <= base_convert($dims[$idx][1], 16, 10)) {
        $key = pack('H*', $key_hex);
        $row_key = pack('H*', $hex);
        if (empty($columns) or array_search($key, $columns) !== FALSE) {
          $values[$key] = isset($row[$row_key]) ? trim($row[$row_key]) : NULL;
        }
        $hex = tpps_increment_hex($hex);
        $key_hex = tpps_increment_hex($key_hex);
      }
    }
    yield $values;
  }

  // Close readers.
  foreach ($readers as $reader) {
    $reader->close();
  }
  tpps_rmdir($dir);
}

function tpps_flat_generator($location, array $options = array()) {
  $original = ini_get('auto_detect_line_endings');
  if (!$original) {
    ini_set('auto_detect_line_endings', TRUE);
  }

  $no_header = $options['no_header'] ?? FALSE;
  $columns = $options['columns'] ?? NULL;
  $max_rows = $options['max_rows'] ?? NULL;

  $delim = ',';
  $longest = tpps_flat_width($location, $delim);
  $handle = gzopen($location, 'r');

  if (!$no_header) {
    gzgets($handle);
  }
  $count = 0;

  while (($line = gzgets($handle))) {
    $vals = str_getcsv($line, $delim);
    if (!empty($max_rows) and $count >= $max_rows) {
      break;
    }
    $count++;

    $values = array();

    if (empty($columns)) {
      $hex = unpack('H*', 'A')[1];
      for ($i = 0; $i < $longest; $i++) {
        $key = pack('H*', $hex);
        $values[$key] = isset($vals[$i]) ? trim($vals[$i]) : NULL;
        if (isset($values[$key]) and !check_plain($values[$key])) {
          $values[$key] = trim(mb_convert_encoding($values[$key], "UTF-8", "Windows-1252"));
        }
        $hex = tpps_increment_hex($hex);
      }
      yield $values;
      continue;
    }

    foreach ($columns as $column) {
      $values[$column] = $vals[tpps_convert_colname($column)] ?? NULL;
    }

    yield $values;
  }

  fclose($handle);
  ini_set('auto_detect_line_endings', $original);
}