11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
|
# File 'manifests/ext/xloader.pp', line 11
class ckan::ext::xloader (
String $formats = 'csv application/csv xls application/vnd.ms-excel',
String $revision = 'master'
) {
ckan::ext { 'xloader':
plugin => ['xloader'],
revision => $revision,
}
$extdir = '/usr/lib/ckan/default/src/ckanext-xloader'
exec { 'install_xloader_security':
command => "${ckan::pip} install -U requests[security]",
refreshonly => true,
cwd => $extdir,
subscribe => Ckan::Ext['xloader'],
}
exec { 'xloader_setup_db':
command => '/usr/bin/psql datastore_default -f full_text_function.sql',
refreshonly => true,
cwd => $extdir,
user => 'postgres',
subscribe => Exec['install_xloader_security'],
}
ckan::conf::setting { 'ckanext.xloader.jobs_db.uri':
value => "postgresql://ckan_default:${ckan::ckan_pass}@localhost/ckan_default",
}
ckan::conf::setting { 'ckanext.xloader.formats':
value => $formats,
}
# The maximum size of files to load into DataStore. In bytes. Default is 1 GB.
ckan::conf::setting { 'ckanext.xloader.max_content_length':
value => '1000000000',
}
# To always use messytables to load data, instead of attempting a direct
# PostgreSQL COPY, set this to True. This more closely matches the
# DataPusher's behavior. It has the advantage that the column types
# are guessed. However it is more error prone, far slower and you can't run
# the CPU-intensive queue on a separate machine.
# ckanext.xloader.just_load_with_messytables = False
# The maximum time for the loading of a resource before it is aborted.
# Give an amount in seconds. Default is 60 minutes
# ckanext.xloader.job_timeout = 3600
# Ignore the file hash when submitting to the DataStore, if set to True
# resources are always submitted (if their format matches), if set to
# False (default), resources are only submitted if their hash has changed.
# ckanext.xloader.ignore_hash = False
# When loading a file that is bigger than `max_content_length`, xloader can
# still try and load some of the file, which is useful to display a
# preview. Set this option to the desired number of lines/rows that it
# loads in this case.
# If the file-type is supported (CSV, TSV) an excerpt with the number of
# `max_excerpt_lines` lines will be submitted while the `max_content_length`
# is not exceeded.
# If set to 0 (default) files that exceed the `max_content_length` will
# not be loaded into the datastore.
# ckanext.xloader.max_excerpt_lines = 100
}
|