diff --git a/assets/xmlimporter.js b/assets/xmlimporter.js
index c373832..89173d6 100644
--- a/assets/xmlimporter.js
+++ b/assets/xmlimporter.js
@@ -18,4 +18,38 @@
}).trigger('change.xmlimporter');
});
+ $(document).on('click','.throttle',function(){
+ var data = {
+ "created":0,
+ "updated":0,
+ "skipped":0,
+ "failed":0,
+ };
+ data.next = $('.next-page').data('next');
+ data.nexturi = $('.next-page').data('uri');
+ throttle(data,0);
+ });
+
+ function throttle(data,maxpage){
+ $importdetails = $('.xmlimport-details');
+ $importdetails.data('created', $importdetails.data('created') + data.created);
+ $importdetails.data('updated', $importdetails.data('updated') + data.updated);
+ $importdetails.data('skipped', $importdetails.data('skipped') + data.skipped);
+ $importdetails.data('failed', $importdetails.data('failed') + data.failed);
+ $importdetails.text( $importdetails.data('created') + ' new entries were created, ' +
+ $importdetails.data('updated') +' updated, ' +
+ $importdetails.data('skipped') +' skipped and ' +
+ $importdetails.data('failed') +' failed'
+ );
+
+ if (data.next != null && (data.next <= maxpage || maxpage == 0)){
+ $importdetails.prev().text('Currently Importing Page ' + data.next);
+ $.getJSON( data.nexturi + '&ajax=1' , function( data ) {
+ throttle(data,maxpage);
+ });
+ } else {
+ $importdetails.prev().text('Import Complete');
+ }
+ }
+
})(jQuery);
diff --git a/content/content.importers.php b/content/content.importers.php
index b552199..14a2699 100644
--- a/content/content.importers.php
+++ b/content/content.importers.php
@@ -276,7 +276,13 @@ public function __viewRun() {
$importer_result['updated'],
$importer_result['skipped'],
$importer_result['failed']
- ))
+ )), array(
+ 'data-created' => $importer_result['created'],
+ 'data-updated' => $importer_result['updated'],
+ 'data-skipped' => $importer_result['skipped'],
+ 'data-failed' => $importer_result['failed'],
+ 'class' => 'xmlimport-details',
+ )
));
// Import errors
@@ -303,9 +309,35 @@ public function __viewRun() {
$importer_result['created'],
$importer_result['updated'],
$importer_result['skipped']
- ))
+ )), array(
+ 'data-created' => $importer_result['created'],
+ 'data-updated' => $importer_result['updated'],
+ 'data-skipped' => $importer_result['skipped'],
+ 'data-failed' => $importer_result['failed'],
+ 'class' => 'xmlimport-details',
+ )
));
+ $throttle = $importer->getThrottle();
+ $pagination = $importer->pagination();
+ if ( !empty($throttle)){
+ $fieldset->appendChild(Widget::Anchor(
+ __('Next Page'),
+ $this->_uri . '/importers/run/' . $this->_context[1] . '/?' . $pagination['variable'] . '=' . $throttle['next-page'],
+ __('Next Page'),
+ 'button next-page',null,array(
+ 'data-next'=>$throttle['next-page'],
+ 'data-uri' => $this->_uri . '/importers/run/' . $this->_context[1] . '/?' . $pagination['variable'] . '=' . $throttle['next-page']
+ )
+ ));
+ $fieldset->appendChild(Widget::Anchor(
+ __('Throttle'),
+ '#throttle',
+ __('Throttle'),
+ 'button throttle'
+ ));
+ }
+
}
$this->Form->appendChild($fieldset);
@@ -323,6 +355,37 @@ public function __viewRun() {
'entries' => $entries
)
);
+
+ $throttle = $importer->getThrottle();
+ $pagination = $importer->pagination();
+ if ($throttle['ajax']){
+
+ $failedEntries = new XMLElement(
+ 'div', null, array('class'=>'failed-entries')
+ );
+
+ if (!empty($failed)){
+ $fieldset->appendChild(new XMLElement(
+ 'h3', __('Import Errors: Page %d', array(
+ $throttle['current-page']
+ ))
+ ));
+
+ $this->addFailedEntries($failedEntries, $failed);
+ }
+
+ echo json_encode( array(
+ 'created' => $importer_result['created'],
+ 'updated' => $importer_result['updated'],
+ 'skipped' => $importer_result['skipped'],
+ 'failed' => $importer_result['failed'],
+ 'next' => $throttle['next-page'],
+ 'nexturi' => $this->_uri . '/importers/run/' . $this->_context[1] . '/?' . $pagination['variable'] . '=' . $throttle['next-page'],
+ 'errors' => $failedEntries->generate()
+ ));
+
+ die;
+ }
}
}
@@ -459,6 +522,18 @@ public function __actionEditNormal() {
: 'no'
);
+ // pagination/throttling settings
+
+ // validate has next page xpath
+ if (!empty($fields['pagination']['next'])){
+ try {
+ $this->_driver->validateXPath($fields['pagination']['next'], $fields['namespaces']);
+ }
+ catch (Exception $e) {
+ $this->_errors['pagination']['next'] = $e->getMessage();
+ }
+ }
+
$this->_fields = $fields;
if (!empty($this->_errors)) {
@@ -669,6 +744,81 @@ public function __viewEdit() {
$this->Form->appendChild($fieldset);
+ // Pagination --------------------------------------------------
+
+
+ $fieldset = new XMLElement('fieldset');
+ $fieldset->setAttribute('class', 'settings');
+ $fieldset->appendChild(new XMLElement('legend', __('Throttling')));
+
+ $group = new XMLElement('div');
+ $group->setAttribute('class', 'two columns');
+
+ $label = Widget::Label(__('Pagination Variable Optional'));
+ $label->setAttribute('class', 'column');
+ $input = Widget::Input(
+ 'fields[pagination][variable]',
+ General::sanitize(
+ isset($this->_fields['pagination']['variable'])
+ ? $this->_fields['pagination']['variable']
+ : null
+ )
+ );
+ $input->setAttribute('placeholder', 'p');
+ $label->appendChild($input);
+
+ if (isset($this->_errors['variable'])) {
+ $label = Widget::Error($label, $this->_errors['variable']);
+ }
+
+ $group->appendChild($label);
+
+ $label = Widget::Label(__('Start Page Optional'));
+ $label->setAttribute('class', 'column');
+ $input = Widget::Input(
+ 'fields[pagination][start]',
+ General::sanitize(
+ isset($this->_fields['pagination']['start'])
+ ? $this->_fields['pagination']['start']
+ : null
+ )
+ );
+ $input->setAttribute('placeholder', '1');
+ $label->appendChild($input);
+
+ $fieldset->appendChild($group);
+
+ $group->appendChild($label);
+ $help = new XMLElement('p');
+ $help->setAttribute('class', 'help');
+ $help->setValue(__('The pagination variable has to be included within your Datasource as a url parameter. Not setting a variable will ignore throttling.'));
+ $fieldset->appendChild($help);
+
+ $label = Widget::Label(__('Next Page Optional'));
+ $input = Widget::Input(
+ 'fields[pagination][next]',
+ General::sanitize(
+ isset($this->_fields['pagination']['next'])
+ ? $this->_fields['pagination']['next']
+ : null
+ )
+ );
+ $input->setAttribute('placeholder', '/data/datasource/pagination/@current-page != /data/datasource/pagination/@total-pages');
+ $label->appendChild($input);
+
+ if (isset($this->_errors['pagination']['next'])) {
+ $label = Widget::Error($label, $this->_errors['pagination']['next']);
+ }
+
+ $fieldset->appendChild($label);
+
+ $help = new XMLElement('p');
+ $help->setAttribute('class', 'help');
+ $help->setValue(__('Use an XPath expression to determine if there is a next page (boolean).'));
+ $fieldset->appendChild($help);
+
+ $this->Form->appendChild($fieldset);
+
// Section ------------------------------------------------------------
$sections = SectionManager::fetch(null, 'ASC', 'name');
@@ -713,6 +863,84 @@ public function __viewEdit() {
if ($fields === false) continue;
+ //entry ID markup
+ $field_id = 'entry-id';
+ // $field_name = "fields[fields][-1]";
+ $field_name = "fields[fields][id]"; //not sure that is correct but should avoid conflicts with other fields
+ $field_data = null;
+ $template_index = null;
+ if (isset($this->_fields['fields'])) {
+ foreach ($this->_fields['fields'] as $i => $temp_data) {
+ if ($temp_data['field'] != $field_id) continue;
+ $field_data = $temp_data;
+ $template_index = $i;
+ // always force the unique to entry id if exists
+ $this->_fields['unique-field'] = "entry-id";
+ }
+ }
+ $li = new XMLElement('li',"Entry ID
",array('class'=>'unique template','data-type'=>'entry-id'));
+ $li->appendChild(new XMLElement('header', 'Entry ID
'));
+
+ $input = Widget::Input("{$field_name}[field]", $field_id, 'hidden');
+ $li->appendChild($input);
+
+ $group = new XMLElement('div');
+ $group->setAttribute('class', 'two columns');
+
+ $label = Widget::Label(__('XPath Expression'));
+ $label->setAttribute('class', 'column');
+ $input = Widget::Input(
+ "{$field_name}[xpath]",
+ General::sanitize(
+ ( isset($field_data) && isset($field_data['xpath']) )
+ ? $field_data['xpath']
+ : null
+ )
+ );
+ $label->appendChild($input);
+ $group->appendChild($label);
+
+ $label = Widget::Label(__('PHP Function'));
+ $label->appendChild(new XMLElement('i', __('Optional')));
+ $label->setAttribute('class', 'column');
+ $input = Widget::Input(
+ "{$field_name}[php]",
+ General::sanitize(
+ ( isset($field_data) && isset($field_data['php']) )
+ ? $field_data['php']
+ : null
+ )
+ );
+ $label->appendChild($input);
+ $group->appendChild($label);
+
+ $li->appendChild($group);
+
+ $label = Widget::Label();
+ $label->setAttribute('class', 'meta');
+
+ $label->setValue(__('Entry ID is used to determine uniqeness.'));
+
+ $li->appendChild($label);
+ $section_fields->appendChild($li);
+
+ if (!is_null($field_data)){
+ //clone to avoid re-setting all the variables
+ $newLi = clone $li;
+ $newLi->setAttribute('class', 'unique');
+
+ //an entry ID must be unique - check only when showing in selected view
+ $input = Widget::Input("fields[unique-field]", $field_id, 'radio');
+ $input->setAttribute("checked","checked");
+ $input->setAttribute("style","display:none");
+ $newLi->appendChild($input);
+
+ //append item to the field list
+ $section_fields->appendChild($newLi);
+ }
+
+ //end entry id
+
// Templates
foreach ($fields as $index => $field) {
$field_id = $field->get('id');
diff --git a/extension.driver.php b/extension.driver.php
index 7a41198..3c45ccd 100644
--- a/extension.driver.php
+++ b/extension.driver.php
@@ -118,6 +118,7 @@ public function getXMLImporter($name) {
$data = $importer->options();
$data['about'] = $importer->about();
+ $data['pagination'] = $importer->pagination();
return $data;
}
@@ -204,7 +205,12 @@ public function setXMLImporter(&$name, &$error, $new) {
var_export($new['source'], true),
var_export($new['timeout'], true),
var_export($new['section'], true),
- var_export($new['unique-field'], true)
+ var_export($new['unique-field'], true),
+
+ // pagination
+ var_export($new['pagination']['variable'], true),
+ var_export($new['pagination']['start'], true),
+ var_export($new['pagination']['next'], true)
);
// Write file to disk:
diff --git a/lib/class.xmlimporter.php b/lib/class.xmlimporter.php
index 956c404..4b036e9 100644
--- a/lib/class.xmlimporter.php
+++ b/lib/class.xmlimporter.php
@@ -25,6 +25,7 @@ class XMLImporter {
protected $_entries = array();
protected $_errors = array();
+ protected $_throttle = array();
public $context = array();
public function about() {
@@ -35,6 +36,14 @@ public function options() {
return array();
}
+ public function pagination() {
+ return array();
+ }
+
+ public function getThrottle() {
+ return $this->_throttle;
+ }
+
public function getEntries() {
return $this->_entries;
}
@@ -86,6 +95,7 @@ function handleXMLError($errno, $errstr, $errfile, $errline, $context) {
$self = $this; // Fucking PHP...
$options = $this->options();
+ $pagination = $this->pagination();
$passed = true;
// If $remote, override the source of the XMLImporter with the given $source
@@ -172,6 +182,27 @@ function handleXMLError($errno, $errstr, $errfile, $errline, $context) {
}
}
+ //pagination
+ if (!empty($pagination['variable'])){
+ // $next = $xpath->query($pagination['next']);
+ $next = $xpath->evaluate('boolean(' . $pagination['next'] . ')');
+
+ $currentPage = intval($pagination['start']);
+ if (isset($this->context['env']['url']['url-' . $pagination['variable']])){
+ $currentPage = intval($this->context['env']['url']['url-' . $pagination['variable']]);
+ }
+
+ $this->_throttle['current-page'] = $currentPage;
+
+ if ($next){
+ $this->_throttle['next-page'] = $currentPage + 1;
+ }
+
+ if (isset($this->context['env']['url']['url-ajax'])){
+ $this->_throttle['ajax'] = true;
+ }
+ }
+
// Invalid Markup:
if (empty($xml)) {
$passed = false;
@@ -251,16 +282,48 @@ function handleXMLError($errno, $errstr, $errfile, $errline, $context) {
$passed = true;
foreach ($this->_entries as $index => &$current) {
+
$entry = EntryManager::create();
$entry->set('section_id', $options['section']);
$entry->set('author_id', is_null(Symphony::Engine()->Author()) ? '1' : Symphony::Engine()->Author()->get('id'));
$entry->set('modification_date_gmt', DateTimeObj::getGMT('Y-m-d H:i:s'));
$entry->set('modification_date', DateTimeObj::get('Y-m-d H:i:s'));
+ //if it has a pre-set entry id use it
+ if(isset($current['values']['entry-id'])){
+ $entry_id = $current['values']['entry-id'];
+ if (is_array($entry_id)){
+ $entry_id = current($entry_id);
+ }
+
+ //entry has an id - should check if it's a new entry
+ $existingEntry = EntryManager::fetch($entry_id, null, null, null, null, null, false, false);
+ if (!empty($existingEntry)){
+ $existingEntry = current($existingEntry);
+ //check that entry is in the same section else there is something wrong
+ if ($entry->get('section_id') != $existingEntry['section_id']){
+ $errorMessage = sprintf(__("An entry with id: %d already exists in a the '%s' Section, and you're trying to import it into the '%s' Section."),
+ $existingEntry['id'],
+ SectionManager::fetch($existingEntry['section_id'])->get('name'),
+ SectionManager::fetch($entry->get('section_id'))->get('name')
+ );
+ $current['errors'] = array($errorMessage);
+ $passed = false;
+ }
+ }
+
+ //set the entry id to continue
+ $entry->set('id', $entry_id);
+ }
+
$values = array();
// Map values:
foreach ($current['values'] as $field_id => $value) {
+ if ($field_id == "entry-id"){
+ continue;
+ }
+
$field = FieldManager::fetch($field_id);
if(is_array($value)) {
@@ -368,6 +431,16 @@ public function commit($status) {
$this->checkExisting($field, $entry, $index, $existing);
};
+ //this entry has an entry id - ignore any existing field previously set and re-set according to this id
+ if ($entry->get('id')){
+ //entry has an id - should check if it's a new entry
+ $existingEntry = EntryManager::fetch($entry->get('id'), null, null, null, null, null, false, false);
+ if (!empty($existingEntry)){
+ $existingEntry = current($existingEntry);
+ $existing[$index] = $existingEntry['id'];
+ }
+ }
+
// Matches an existing entry
if (!is_null($existing[$index])) {
// Update
diff --git a/templates/xml-importer.php b/templates/xml-importer.php
index f9f4ead..285d393 100644
--- a/templates/xml-importer.php
+++ b/templates/xml-importer.php
@@ -33,6 +33,15 @@ public function options()
);
}
+ public function pagination()
+ {
+ return array(
+ 'variable' => %s,
+ 'start' => %s,
+ 'next' => %s,
+ );
+ }
+
public function allowEditorToParse()
{
return true;