diff options
author | lookshe <lookshe@lookshe-laptop.(none)> | 2014-09-21 00:39:52 +0200 |
---|---|---|
committer | lookshe <lookshe@lookshe-laptop.(none)> | 2014-09-21 00:39:52 +0200 |
commit | 61672bf575c84e1f3b313fe9ea2dd4de75b601ae (patch) | |
tree | e42d542afd5de8ead38bdc42a002fe0c336b6b75 | |
parent | 404e0e28825c805b54d129f8b9d7c5690a27ef17 (diff) | |
parent | 9dd1e786b135cbf333e861f790eb53091904a607 (diff) |
Merge branch 'master' of github.com:lookshe/scripts
-rw-r--r-- | download_webscraper.pl | 35 | ||||
-rw-r--r-- | externalIp.pl | 6 | ||||
-rw-r--r-- | ipchanges.pl | 21 | ||||
-rw-r--r-- | osm/config.cfg | 1272 | ||||
-rw-r--r-- | osm/config.cfg_bak | 53 | ||||
-rwxr-xr-x | osm/download_and_convert.sh | 68 | ||||
-rwxr-xr-x | osm/main.sh | 50 |
7 files changed, 1505 insertions, 0 deletions
diff --git a/download_webscraper.pl b/download_webscraper.pl new file mode 100644 index 0000000..a8bd1ea --- /dev/null +++ b/download_webscraper.pl @@ -0,0 +1,35 @@ +#!/usr/bin/perl + +#use strict; +#use warnings; +use Web::Scraper; +use URI; +use HTML::Entities; +use Encode; +use URI::Escape; +use LWP::UserAgent; + +my $scrap; + +my $wikiurl = "http://www.bildung-lsa.de/unterricht/zentrale_leistungserhebungen__schriftliche_pruefungen__zentrale_klassenarbeiten__vergleichsarbeiten____/schriftliche_abiturpruefung.html"; + +my $ua = new LWP::UserAgent; +my $req = HTTP::Request->new('GET', $wikiurl); +my $res = $ua->request($req); +my $url = $res->request->uri; + +binmode(STDOUT, ":utf8"); + + + $scrap = scraper { + process '//a[@class="subjectlink"]', 'href[]' => '@href'; + }; + $url = URI->new($wikiurl); + + my $res = $scrap->scrape($url); + my $href = $res->{'href'}; + for ($i = 0; $i <= $#$href; $i++) + { + my $url = $$href[$i]; + system("wget -q \"$url\""); + } diff --git a/externalIp.pl b/externalIp.pl new file mode 100644 index 0000000..2220d9d --- /dev/null +++ b/externalIp.pl @@ -0,0 +1,6 @@ +#!/usr/bin/perl + +($ip=`wget -q -O - http://checkip.dyndns.org`)=~s/\n$//; +$ip=~s/.*[^\d](\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[^\d].*/$1/; +($date=`date`)=~s/\n$//; +print "$date\n$ip\n"; diff --git a/ipchanges.pl b/ipchanges.pl new file mode 100644 index 0000000..5b3211a --- /dev/null +++ b/ipchanges.pl @@ -0,0 +1,21 @@ +#!/usr/bin/perl + +my $file = "/home/lookshe/myips.txt"; + +open(FILE, $file) || die("unable to open $file!"); +my @lines = <FILE>; +close(FILE); + +for ($i = 1; $i <= $#lines-2; $i+=2) +{ + my $ip = $lines[$i]; + chomp($ip); + my $nextip = $lines[$i+2]; + chomp($nextip); + if ($ip !~ m/^$nextip$/) + { + my $time = $lines[$i+1]; + chomp($time); + print "$time: $ip -> $nextip\n"; + } +} diff --git a/osm/config.cfg b/osm/config.cfg new file mode 100644 index 0000000..7f3333c --- /dev/null +++ b/osm/config.cfg @@ -0,0 +1,1272 @@ +[general] +osmosis_executable=/home/osm/convert/tools/osmosis-0.40.1/bin/osmosis +download_and_convert_script=/home/osm/convert/scripts/download_and_convert.sh + +dir_maps=/home/osm/convert/maps +dir_output=/home/osm/convert/output +dir_stage=/home/osm/convert/stage +dir_poly=/home/osm/convert/poly + +default_type=ram +default_start_zoom=14 +default_language=en +default_use_poly=true +filetype_osm=osm.pbf +filetype_map=map +filetype_poly=poly + +threads_to_start=1 + +download_base_url=http://download.geofabrik.de +wget_limit_rate=20m + +maps_to_generate="baden-wuerttemberg bayern berlin brandenburg bremen hamburg hessen mecklenburg-vorpommern niedersachsen nordrhein-westfalen rheinland-pfalz saarland sachsen sachsen-anhalt schleswig-holstein thueringen albania andorra austria azores belarus belgium bosnia-herzegovina bulgaria croatia cyprus czech-republic denmark estonia faroe-islands finland great-britain greece hungary iceland ireland-and-northern-ireland isle-of-man italy kosovo latvia liechtenstein lithuania luxembourg macedonia malta moldova monaco montenegro netherlands norway poland portugal romania russia-european-part serbia slovakia slovenia spain sweden switzerland turkey urkaine alsace aquitaine auvergne basse-normandie bourgogne bretagne centre champagne-ardenne corse franche-comte guadeloupe guyane haute-normandie ile-de-france languedoc-roussillon limousin lorraine martinique mayotte midi-pyrenees nord-pas-de-calais pays-de-la-loire picardie poitou-charentes provence-alpes-cote-d-azur reunion rhone-alpes botswana burkina-faso cameroon canary-islands congo-democratic-republic egypt ethiopia guinea guinea-bissau ivory-coast liberia libya madagascar morocco nigeria sierra-leone somalia south-africa-and-lesotho tanzania azerbaijan bangladesh china gcc-states india indonesia iran iraq israel-and-palestine japan jordan kazakhstan lebanon malaysia-singapore-brunei mongolia pakistan philippines russia-asian-part sri-lanka waiwan tajikistan thailand turkmenistan uzbekistan vietnam australia fiji new-caledonia new-zealand belize cuba guatemala haiti-and-domrep mexico antarctica argentina bolivia brazil chile colombia ecuador peru uruguay canada greenland alabama alaska arizona arkansas california colorado connecticut delaware district-of-columbia florida georgia hawaii idaho illinois indiana iowa kansas kentucky louisiana maine maryland massachusetts michigan minnesota mississippi missouri montana nebraska nevada new-hampshire new-jersey new-mexico new-york noth-carolina north-dakota ohio oklahoma oregon pennsylcania rhode-island south-carolina south-dakota tennessee texas utah vermont virginia washington west-virginia wisconsin wyoming" +[/general] + +[possible_values] +download_map_path=relative path to map-file to downloaded +download_map_file=filename of the map-file to download +download_poly_path=relativ path to poly-file to download +download_poly_file=filename of poly-file to download +output_subdir=subdir for better sorting +use_poly=override $default_use_poly +wm_type=overwrite $default_type (hd|ram) +start_zoom=overwrite $default_start_zoom +language=override $default_language +[/possible_values] + +#[germany] +#download_map_path=europe +#download_map_file=germany-latest +#download_poly_path=europe +#download_poly_file=germany +#use_poly=false +#output_subdir=europe +#[/germany] + +[baden-wuerttemberg] +download_map_path=europe/germany +download_map_file=baden-wuerttemberg-latest +download_poly_path=europe/germany +download_poly_file=baden-wuerttemberg +output_subdir=europe/germany +use_poly=false +wm_type=hd +[/baden-wuerttemberg] + +[bayern] +download_map_file=bayern-latest +download_poly_file=bayern +use_poly=false +wm_type=hd +[/bayern] + +[berlin] +download_map_file=berlin-latest +download_poly_file=berlin +use_poly=false +[/berlin] + +[brandenburg] +download_map_file=brandenburg-latest +download_poly_file=brandenburg +use_poly=false +[/brandenburg] + +[bremen] +download_map_file=bremen-latest +download_poly_file=bremen +use_poly=false +[/bremen] + +[hamburg] +download_map_file=hamburg-latest +download_poly_file=hamburg +use_poly=false +[/hamburg] + +[hessen] +download_map_file=hessen-latest +download_poly_file=hessen +use_poly=false +wm_type=hd +[/hessen] + +[mecklenburg-vorpommern] +download_map_file=mecklenburg-vorpommern-latest +download_poly_file=mecklenburg-vorpommern +use_poly=false +[/mecklenburg-vorpommern] + +[niedersachsen] +download_map_file=niedersachsen-latest +download_poly_file=niedersachsen +use_poly=false +wm_type=hd +[/niedersachsen] + +[nordrhein-westfalen] +download_map_file=nordrhein-westfalen-latest +download_poly_file=nordrhein-westfalen +use_poly=false +wm_type=hd +[/nordrhein-westfalen] + +[rheinland-pfalz] +download_map_file=rheinland-pfalz-latest +download_poly_file=rheinland-pfalz +use_poly=false +[/rheinland-pfalz] + +[saarland] +download_map_file=saarland-latest +download_poly_file=saarland +use_poly=false +[/saarland] + +[sachsen] +download_map_file=sachsen-latest +download_poly_file=sachsen +use_poly=false +[/sachsen] + +[sachsen-anhalt] +download_map_file=sachsen-anhalt-latest +download_poly_file=sachsen-anhalt +use_poly=false +[/sachsen-anhalt] + +[schleswig-holstein] +download_map_file=schleswig-holstein-latest +download_poly_file=schleswig-holstein +use_poly=false +[/schleswig-holstein] + +[thueringen] +download_map_file=thueringen-latest +download_poly_file=thueringen +use_poly=false +[/thueringen] + +[albania] +download_map_path=europe +download_map_file=albania-latest +download_poly_path=europe +download_poly_file=albania +use_poly=false +output_subdir=europe +[/albania] + +[andorra] +download_map_file=andorra-latest +download_poly_file=andorra +use_poly=false +[/andorra] + +[austria] +download_map_file=austria-latest +download_poly_file=austria +use_poly=false +wm_type=hd +[/austria] + +[azores] +download_map_file=azores-latest +download_poly_file=azores +use_poly=false +[/azores] + +[belarus] +download_map_file=belarus-latest +download_poly_file=belarus +use_poly=false +[/belarus] + +[belgium] +download_map_file=belgium-latest +download_poly_file=belgium +use_poly=false +wm_type=hd +[/belgium] + +[bosnia-herzegovina] +download_map_file=bosnia-herzegovina-latest +download_poly_file=bosnia-herzegovina +use_poly=false +wm_type=hd +[/bosnia-herzegovina] + +[bulgaria] +download_map_file=bulgaria-latest +download_poly_file=bulgaria +use_poly=false +[/bulgaria] + +[croatia] +download_map_file=croatia-latest +download_poly_file=croatia +use_poly=false +[/croatia] + +[cyprus] +download_map_file=cyprus-latest +download_poly_file=cyprus +use_poly=false +[/cyprus] + +[czech-republic] +download_map_file=czech-republic-latest +download_poly_file=czech-republic +use_poly=false +wm_type=hd +[/czech-republic] + +[denmark] +download_map_file=denmark-latest +download_poly_file=denmark +use_poly=false +wm_type=hd +[/denmark] + +[estonia] +download_map_file=estonia-latest +download_poly_file=estonia +use_poly=false +[/estonia] + +[faroe-islands] +download_map_file=faroe-islands-latest +download_poly_file=faroe-islands +use_poly=false +[/faroe-islands] + +[finland] +download_map_file=finland-latest +download_poly_file=finland +use_poly=false +wm_type=hd +[/finland] + +[great-britain] +download_map_file=great-britain-latest +download_poly_file=great-britain +use_poly=false +wm_type=hd +[/great-britain] + +[greece] +download_map_file=greece-latest +download_poly_file=greece +use_poly=false +[/greece] + +[hungary] +download_map_file=hungary-latest +download_poly_file=hungary +use_poly=false +[/hungary] + +[iceland] +download_map_file=iceland-latest +download_poly_file=iceland +use_poly=false +[/iceland] + +[ireland-and-northern-ireland] +download_map_file=ireland-and-northern-ireland-latest +download_poly_file=ireland-and-northern-ireland +use_poly=false +[/ireland-and-northern-ireland] + +[isle-of-man] +download_map_file=isle-of-man-latest +download_poly_file=isle-of-man +use_poly=false +[/isle-of-man] + +[italy] +download_map_file=italy-latest +download_poly_file=italy +use_poly=false +wm_type=hd +[/italy] + +[kosovo] +download_map_file=kosovo-latest +download_poly_file=kosovo +use_poly=false +[/kosovo] + +[latvia] +download_map_file=latvia-latest +download_poly_file=latvia +use_poly=false +[/latvia] + +[liechtenstein] +download_map_file=liechtenstein-latest +download_poly_file=liechtenstein +use_poly=false +[/liechtenstein] + +[lithuania] +download_map_file=lithuania-latest +download_poly_file=lithuania +use_poly=false +[/lithuania] + +[luxembourg] +download_map_file=luxembourg-latest +download_poly_file=luxembourg +use_poly=false +[/luxembourg] + +[macedonia] +download_map_file=macedonia-latest +download_poly_file=macedonia +use_poly=false +[/macedonia] + +[malta] +download_map_file=malta-latest +download_poly_file=malta +use_poly=false +[/malta] + +[moldova] +download_map_file=moldova-latest +download_poly_file=moldova +use_poly=false +[/moldova] + +[monaco] +download_map_file=monaco-latest +download_poly_file=monaco +use_poly=false +[/monaco] + +[montenegro] +download_map_file=montenegro-latest +download_poly_file=montenegro +use_poly=false +[/montenegro] + +[netherlands] +download_map_file=netherlands-latest +download_poly_file=netherlands +use_poly=false +wm_type=hd +[/netherlands] + +[norway] +download_map_file=norway-latest +download_poly_file=norway +use_poly=false +wm_type=hd +[/norway] + +[poland] +download_map_file=poland-latest +download_poly_file=poland +use_poly=false +wm_type=hd +[/poland] + +[portugal] +download_map_file=portugal-latest +download_poly_file=portugal +use_poly=false +[/portugal] + +[romania] +download_map_file=romania-latest +download_poly_file=romania +use_poly=false +wm_type=hd +[/romania] + +[russia-european-part] +download_map_file=russia-european-part-latest +download_poly_file=russia-european-part +use_poly=false +[/russia-european-part] + +[serbia] +download_map_file=serbia-latest +download_poly_file=serbia +use_poly=false +[/serbia] + +[slovakia] +download_map_file=slovakia-latest +download_poly_file=slovakia +use_poly=false +wm_type=hd +[/slovakia] + +[slovenia] +download_map_file=slovenia-latest +download_poly_file=slovenia +use_poly=false +[/slovenia] + +[spain] +download_map_file=spain-latest +download_poly_file=spain +use_poly=false +wm_type=hd +[/spain] + +[sweden] +download_map_file=sweden-latest +download_poly_file=sweden +use_poly=false +wm_type=hd +[/sweden] + +[switzerland] +download_map_file=switzerland-latest +download_poly_file=switzerland +use_poly=false +wm_type=hd +[/switzerland] + +[turkey] +download_map_file=turkey-latest +download_poly_file=turkey +use_poly=false +wm_type=hd +[/turkey] + +[urkaine] +download_map_file=urkaine-latest +download_poly_file=urkaine +use_poly=false +wm_type=hd +[/urkaine] + +#[france] +#download_map_file=france-latest +#download_poly_file=france +#use_poly=false +#[/france] + +[alsace] +download_map_path=europe/france +download_map_file=alsace-latest +download_poly_path=europe/france +download_poly_file=alsace +output_subdir=europe/france +use_poly=false +[/alsace] + +[aquitaine] +download_map_file=aquitaine-latest +download_poly_file=aquitaine +use_poly=false +wm_type=hd +[/aquitaine] + +[auvergne] +download_map_file=auvergne-latest +download_poly_file=auvergne +use_poly=false +[/auvergne] + +[basse-normandie] +download_map_file=basse-normandie-latest +download_poly_file=basse-normandie +use_poly=false +[/basse-normandie] + +[bourgogne] +download_map_file=bourgogne-latest +download_poly_file=bourgogne +use_poly=false +[/bourgogne] + +[bretagne] +download_map_file=bretagne-latest +download_poly_file=bretagne +use_poly=false +wm_type=hd +[/bretagne] + +[centre] +download_map_file=centre-latest +download_poly_file=centre +use_poly=false +[/centre] + +[champagne-ardenne] +download_map_file=champagne-ardenne-latest +download_poly_file=champagne-ardenne +use_poly=false +[/champagne-ardenne] + +[corse] +download_map_file=corse-latest +download_poly_file=corse +use_poly=false +[/corse] + +[franche-comte] +download_map_file=franche-comte-latest +download_poly_file=franche-comte +use_poly=false +[/franche-comte] + +[guadeloupe] +download_map_file=guadeloupe-latest +download_poly_file=guadeloupe +use_poly=false +[/guadeloupe] + +[guyane] +download_map_file=guyane-latest +download_poly_file=guyane +use_poly=false +[/guyane] + +[haute-normandie] +download_map_file=haute-normandie-latest +download_poly_file=haute-normandie +use_poly=false +[/haute-normandie] + +[ile-de-france] +download_map_file=ile-de-france-latest +download_poly_file=ile-de-france +use_poly=false +wm_type=hd +[/ile-de-france] + +[languedoc-roussillon] +download_map_file=languedoc-roussillon-latest +download_poly_file=languedoc-roussillon +use_poly=false +wm_type=hd +[/languedoc-roussillon] + +[limousin] +download_map_file=limousin-latest +download_poly_file=limousin +use_poly=false +[/limousin] + +[lorraine] +download_map_file=lorraine-latest +download_poly_file=lorraine +use_poly=false +[/lorraine] + +[martinique] +download_map_file=martinique-latest +download_poly_file=martinique +use_poly=false +[/martinique] + +[mayotte] +download_map_file=mayotte-latest +download_poly_file=mayotte +use_poly=false +[/mayotte] + +[midi-pyrenees] +download_map_file=midi-pyrenees-latest +download_poly_file=midi-pyrenees +use_poly=false +wm_type=hd +[/midi-pyrenees] + +[nord-pas-de-calais] +download_map_file=nord-pas-de-calais-latest +download_poly_file=nord-pas-de-calais +use_poly=false +wm_type=hd +[/nord-pas-de-calais] + +[pays-de-la-loire] +download_map_file=pays-de-la-loire-latest +download_poly_file=pays-de-la-loire +use_poly=false +wm_type=hd +[/pays-de-la-loire] + +[picardie] +download_map_file=picardie-latest +download_poly_file=picardie +use_poly=false +[/picardie] + +[poitou-charentes] +download_map_file=poitou-charentes-latest +download_poly_file=poitou-charentes +use_poly=false +wm_type=hd +[/poitou-charentes] + +[provence-alpes-cote-d-azur] +download_map_file=provence-alpes-cote-d-azur-latest +download_poly_file=provence-alpes-cote-d-azur +use_poly=false +wm_type=hd +[/provence-alpes-cote-d-azur] + +[reunion] +download_map_file=reunion-latest +download_poly_file=reunion +use_poly=false +[/reunion] + +[rhone-alpes] +download_map_file=rhone-alpes-latest +download_poly_file=rhone-alpes +use_poly=false +wm_type=hd +[/rhone-alpes] + +[botswana] +download_map_path= +download_map_file=africa-latest +download_poly_path=africa +download_poly_file=botswana +output_subdir=africa +[/botswana] + +[burkina-faso] +download_poly_file=burkina-faso +[/burkina-faso] + +[cameroon] +download_poly_file=cameroon +[/cameroon] + +[canary-islands] +download_poly_file=canary-islands +[/canary-islands] + +[congo-democratic-republic] +download_poly_file=congo-democratic-republic +[/congo-democratic-republic] + +[egypt] +download_poly_file=egypt +[/egypt] + +[ethiopia] +download_poly_file=ethiopia +[/ethiopia] + +[guinea] +download_poly_file=guinea +[/guinea] + +[guinea-bissau] +download_poly_file=guinea-bissau +[/guinea-bissau] + +[ivory-coast] +download_poly_file=ivory-coast +[/ivory-coast] + +[liberia] +download_poly_file=liberia +[/liberia] + +[libya] +download_poly_file=libya +[/libya] + +[madagascar] +download_poly_file=madagascar +[/madagascar] + +[morocco] +download_poly_file=morocco +[/morocco] + +[nigeria] +download_poly_file=nigeria +[/nigeria] + +[sierra-leone] +download_poly_file=sierra-leone +[/sierra-leone] + +[somalia] +download_poly_file=somalia +[/somalia] + +[south-africa-and-lesotho] +download_poly_file=south-africa-and-lesotho +[/south-africa-and-lesotho] + +[tanzania] +download_poly_file=tanzania +[/tanzania] + +[azerbaijan] +download_map_path=asia +download_map_file=azerbaijan-latest +download_poly_path=asia +download_poly_file=azerbaijan +output_subdir=asia +use_poly=false +[/azerbaijan] + +[bangladesh] +download_map_file=bangladesh-latest +download_poly_file=bangladesh +use_poly=false +[/bangladesh] + +[china] +download_map_file=china-lastest +download_poly_file=china +use_poly=false +wm_type=hd +[/china] + +[gcc-states] +download_map_file=gcc-states-latest +download_poly_file=gcc-states +use_poly=false +[/gcc-states] + +[india] +download_map_file=india-latest +download_poly_file=india +use_poly=false +[/india] + +[indonesia] +download_map_file=indonesia-latest +download_poly_file=indonesia +use_poly=false +[/indonesia] + +[iran] +download_map_file=iran-latest +download_poly_file=iran +use_poly=false +[/iran] + +[iraq] +download_map_file=iraq-latest +download_poly_file=iraq +use_poly=false +[/iraq] + +[israel-and-palestine] +download_map_file=israel-and-palestine-latest +download_poly_file=israel-and-palestine +use_poly=false +[/israel-and-palestine] + +[japan] +download_map_file=japan-latest +download_poly_file=japan +use_poly=false +wm_type=hd +[/japan] + +[jordan] +download_map_file=jordan-latest +download_poly_file=jordan +use_poly=false +[/jordan] + +[kazakhstan] +download_map_file=kazakhstan-latest +download_poly_file=kazakhstan +use_poly=false +[/kazakhstan] + +[lebanon] +download_map_file=lebanon-latest +download_poly_file=lebanon +use_poly=false +[/lebanon] + +[malaysia-singapore-brunei] +download_map_file=malaysia-singapore-brunei-latest +download_poly_file=malaysia-singapore-brunei +use_poly=false +[/malaysia-singapore-brunei] + +[mongolia] +download_map_file=mongolia-latest +download_poly_file=mongolia +use_poly=false +[/mongolia] + +[pakistan] +download_map_file=pakistan-latest +download_poly_file=pakistan +use_poly=false +[/pakistan] + +[philippines] +download_map_file=philippines-latest +download_poly_file=philippines +use_poly=false +[/philippines] + +[russia-asian-part] +download_map_file=russia-asian-part-latest +download_poly_file=russia-asian-part +use_poly=false +wm_type=hd +[/russia-asian-part] + +[sri-lanka] +download_map_file=sri-lanka-latest +download_poly_file=sri-lanka +use_poly=false +[/sri-lanka] + +[waiwan] +download_map_file=waiwan-latest +download_poly_file=waiwan +use_poly=false +[/waiwan] + +[tajikistan] +download_map_file=tajikistan-latest +download_poly_file=tajikistan +use_poly=false +[/tajikistan] + +[thailand] +download_map_file=thailand-latest +download_poly_file=thailand +use_poly=false +[/thailand] + +[turkmenistan] +download_map_file=turkmenistan-latest +download_poly_file=turkmenistan +use_poly=false +[/turkmenistan] + +[uzbekistan] +download_map_file=uzbekistan-latest +download_poly_file=uzbekistan +use_poly=false +[/uzbekistan] + +[vietnam] +download_map_file=vietnam-latest +download_poly_file=vietnam +use_poly=false +[/vietnam] + +[australia] +download_map_path= +download_map_file=australia-oceania-latest +download_poly_path=australia-oceania +download_poly_file=australia +[/australia] + +[fiji] +download_poly_file=fiji +[/fiji] + +[new-caledonia] +download_poly_file=new-caledonia +[/new-caledonia] + +[new-zealand] +download_poly_file=new-zealand +[/new-zealand] + +[belize] +download_map_path= +download_map_file=central-america-latest +download_poly_path=central-america +download_poly_file=belize +[/belize] + +[cuba] +download_poly_file=cuba +[/cuba] + +[guatemala] +download_poly_file=guatemala +[/guatemala] + +[haiti-and-domrep] +download_poly_file=haiti-and-domrep +[/haiti-and-domrep] + +[mexico] +download_poly_file=mexico +[/mexico] + +[antarctica] +download_map_path= +download_map_file=antarctica-latest +download_poly_path= +download_poly_file=antarctica +use_poly=false +[/antarctica] + +[argentina] +download_map_path= +download_map_file=south-america-latest +download_poly_path=south-america +download_poly_file=argentina +[/argentina] + +[bolivia] +download_poly_file=bolivia +[/bolivia] + +[brazil] +download_poly_file=brazil +[/brazil] + +[chile] +download_poly_file=chile +[/chile] + +[colombia] +download_poly_file=colombia +[/colombia] + +[ecuador] +download_poly_file=ecuador +[/ecuador] + +[peru] +download_poly_file=peru +[/peru] + +[uruguay] +download_poly_file=uruguay +[/uruguay] + +[canada] +download_map_path=north-america +download_map_file=canada-latest +download_poly_path=north-america +download_poly_file=canada +use_poly=false +type=hd +[/canada] + +[greenland] +download_map_file=greenland-latest +download_poly_file=greenland +use_poly=false +[/greenland] + +[alabama] +download_map_path=north-america/us/ +download_map_file=alabama-latest +download_map_path=north-america/us/ +download_poly_file=alabama +use_poly=false +[/alabama] + +[alaska] +download_map_file=alaska-latest +download_poly_file=alaska +use_poly=false +[/alaska] + +[arizona] +download_map_file=arizona-latest +download_poly_file=arizona +use_poly=false +[/arizona] + +[arkansas] +download_map_file=arkansas-latest +download_poly_file=arkansas +use_poly=false +[/arkansas] + +[california] +download_map_file=california-latest +download_poly_file=california +use_poly=false +[/california] + +[colorado] +download_map_file=colorado-latest +download_poly_file=colorado +use_poly=false +[/colorado] + +[connecticut] +download_map_file=connecticut-latest +download_poly_file=connecticut +use_poly=false +[/connecticut] + +[delaware] +download_map_file=delaware-latest +download_poly_file=delaware +use_poly=false +[/delaware] + +[district-of-columbia] +download_map_file=district-of-columbia-latest +download_poly_file=district-of-columbia +use_poly=false +[/district-of-columbia] + +[florida] +download_map_file=florida-latest +download_poly_file=florida +use_poly=false +[/florida] + +[georgia] +download_map_file=georgia-latest +download_poly_file=georgia +use_poly=false +[/georgia] + +[hawaii] +download_map_file=hawaii-latest +download_poly_file=hawaii +use_poly=false +[/hawaii] + +[idaho] +download_map_file=idaho-latest +download_poly_file=idaho +use_poly=false +[/idaho] + +[illinois] +download_map_file=illinois-latest +download_poly_file=illinois +use_poly=false +[/illinois] + +[indiana] +download_map_file=indiana-latest +download_poly_file=indiana +use_poly=false +[/indiana] + +[iowa] +download_map_file=iowa-latest +download_poly_file=iowa +use_poly=false +[/iowa] + +[kansas] +download_map_file=kansas-latest +download_poly_file=kansas +use_poly=false +[/kansas] + +[kentucky] +download_map_file=kentucky-latest +download_poly_file=kentucky +use_poly=false +[/kentucky] + +[louisiana] +download_map_file=louisiana-latest +download_poly_file=louisiana +use_poly=false +[/louisiana] + +[maine] +download_map_file=maine-latest +download_poly_file=maine +use_poly=false +[/maine] + +[maryland] +download_map_file=maryland-latest +download_poly_file=maryland +use_poly=false +[/maryland] + +[massachusetts] +download_map_file=massachusetts-latest +download_poly_file=massachusetts +use_poly=false +[/massachusetts] + +[michigan] +download_map_file=michigan-latest +download_poly_file=michigan +use_poly=false +[/michigan] + +[minnesota] +download_map_file=minnesota-latest +download_poly_file=minnesota +use_poly=false +[/minnesota] + +[mississippi] +download_map_file=mississippi-latest +download_poly_file=mississippi +use_poly=false +[/mississippi] + +[missouri] +download_map_file=missouri-latest +download_poly_file=missouri +use_poly=false +[/missouri] + +[montana] +download_map_file=montana-latest +download_poly_file=montana +use_poly=false +[/montana] + +[nebraska] +download_map_file=nebraska-latest +download_poly_file=nebraska +use_poly=false +[/nebraska] + +[nevada] +download_map_file=nevada-latest +download_poly_file=nevada +use_poly=false +[/nevada] + +[new-hampshire] +download_map_file=new-hampshire-latest +download_poly_file=new-hampshire +use_poly=false +[/new-hampshire] + +[new-jersey] +download_map_file=new-jersey-latest +download_poly_file=new-jersey +use_poly=false +[/new-jersey] + +[new-mexico] +download_map_file=new-mexico-latest +download_poly_file=new-mexico +use_poly=false +[/new-mexico] + +[new-york] +download_map_file=new-york-latest +download_poly_file=new-york +use_poly=false +[/new-york] + +[noth-carolina] +download_map_file=noth-carolina-latest +download_poly_file=noth-carolina +use_poly=false +[/noth-carolina] + +[north-dakota] +download_map_file=north-dakota-latest +download_poly_file=north-dakota +use_poly=false +[/north-dakota] + +[ohio] +download_map_file=ohio-latest +download_poly_file=ohio +use_poly=false +[/ohio] + +[oklahoma] +download_map_file=oklahoma-latest +download_poly_file=oklahoma +use_poly=false +[/oklahoma] + +[oregon] +download_map_file=oregon-latest +download_poly_file=oregon +use_poly=false +[/oregon] + +[pennsylcania] +download_map_file=pennsylcania-latest +download_poly_file=pennsylcania +use_poly=false +[/pennsylcania] + +[rhode-island] +download_map_file=rhode-island-latest +download_poly_file=rhode-island +use_poly=false +[/rhode-island] + +[south-carolina] +download_map_file=south-carolina-latest +download_poly_file=south-carolina +use_poly=false +[/south-carolina] + +[south-dakota] +download_map_file=south-dakota-latest +download_poly_file=south-dakota +use_poly=false +[/south-dakota] + +[tennessee] +download_map_file=tennessee-latest +download_poly_file=tennessee +use_poly=false +[/tennessee] + +[texas] +download_map_file=texas-latest +download_poly_file=texas +use_poly=false +[/texas] + +[utah] +download_map_file=utah-latest +download_poly_file=utah +use_poly=false +[/utah] + +[vermont] +download_map_file=vermont-latest +download_poly_file=vermont +use_poly=false +[/vermont] + +[virginia] +download_map_file=virginia-latest +download_poly_file=virginia +use_poly=false +[/virginia] + +[washington] +download_map_file=washington-latest +download_poly_file=washington +use_poly=false +[/washington] + +[west-virginia] +download_map_file=west-virginia-latest +download_poly_file=west-virginia +use_poly=false +[/west-virginia] + +[wisconsin] +download_map_file=wisconsin-latest +download_poly_file=wisconsin +use_poly=false +[/wisconsin] + +[wyoming] +download_map_file=wyoming-latest +download_poly_file=wyoming +use_poly=false +[/wyoming] + diff --git a/osm/config.cfg_bak b/osm/config.cfg_bak new file mode 100644 index 0000000..b0b5366 --- /dev/null +++ b/osm/config.cfg_bak @@ -0,0 +1,53 @@ +[general] +osmosis_executable=/home/osm/convert/tools/osmosis-0.40.1/bin/osmosis +download_and_convert_script=/home/osm/convert/scripts/download_and_convert.sh + +dir_maps=/home/osm/convert/maps +dir_output=/home/osm/convert/output +dir_stage=/home/osm/convert/stage +dir_poly=/home/osm/convert/poly + +default_type=hd +default_start_zoom=14 +default_language=en +default_use_poly=true +filetype_osm=osm.pbf +filetype_map=map +filetype_poly=poly + +threads_to_start=2 + +download_base_url=http://download.geofabrik.de +wget_limit_rate=20m + +maps_to_generate="germany berlin hamburg" +[/general] + +[possible_values] +download_map_path=relative path to map-file to downloaded +download_map_file=filename of the map-file to download +download_poly_path=relativ path to poly-file to download +download_poly_file=filename of poly-file to download +use_poly=override $default_use_poly +type=overwrite $default_type (hd|ram) +start_zoom=overwrite $default_start_zoom +language=override $default_language +[/possible_values] + +[germany] +download_map_path=europe +download_map_file=germany-latest +download_poly_path=europe +download_poly_file=germany +use_poly=false +[/germany] + +[berlin] +download_poly_path=europe/germany +download_poly_file=berlin +[/berlin] + +[hamburg] +download_poly_path=europe/germany +download_poly_file=hamburg +[/hamburg] diff --git a/osm/download_and_convert.sh b/osm/download_and_convert.sh new file mode 100755 index 0000000..d5c4d5a --- /dev/null +++ b/osm/download_and_convert.sh @@ -0,0 +1,68 @@ +#!/bin/bash +#set -x + +if [ $# -ne 8 ] +then + echo "wrong argument count" + exit 1 +fi + +url_map="$1" +url_map_filename="$(basename $url_map)" +download_map="$dir_maps/$url_map_filename" +url_poly="$2" +url_poly_filename="$(basename $url_poly)" +download_poly="$dir_poly/$url_poly_filename" +mapfile="$3" +wm_type="$4" +start_zoom="$5" +language="$6" +use_poly="$7" +output_subdir="$8" +stage_map="$dir_stage/$mapfile" +mkdir -p "$dir_output/$output_subdir/" +output_map="$dir_output/$output_subdir/$mapfile" + + +# cause of some timing problems we wait a random number of 1-10 seconds +sleep $[($RANDOM % 10) + 1] + +# check if we need to download +if [ ! -e "$download_map" ] +then + # wait if another process is downloading it + if [ -e "$download_map.tmp" ] + then + while [ ! -e "$download_map" ] + do + sleep 10 + done + # download if we are the only one + else + touch "$download_map.tmp" + wget -q --limit-rate=$wget_limit_rate "$url_map" -O "$download_map.tmp" + mv -f "$download_map.tmp" "$download_map" + fi +fi + +# download poly-file +wget -q --limit-rate=$wget_limit_rate "$url_poly" -O "$download_poly" + +echo "start: $(date)" >> "$stage_map.time" +if [ "$use_poly" = "true" ] +then + echo "command: $osmosis_executable --rb \"$download_map\" --bp clipIncompleteEntities=true file=\"$download_poly\" --mw file=\"$stage_map\" type=\"$wm_type\" map-start-zoom=\"$start_zoom\" preferred-language=\"$language\"" >> "$stage_map.time" + $osmosis_executable --rb "$download_map" --bp clipIncompleteEntities=true file="$download_poly" --mw file="$stage_map" type="$wm_type" map-start-zoom="$start_zoom" preferred-language="$language" > "$stage_map.log" 2>&1 + ret=$? +else + echo "command: $osmosis_executable --rb \"$download_map\" --mw file=\"$stage_map\" type=\"$wm_type\" map-start-zoom=\"$start_zoom\" preferred-language=\"$language\"" >> "$stage_map.time" + $osmosis_executable --rb "$download_map" --mw file="$stage_map" type="$wm_type" map-start-zoom="$start_zoom" preferred-language="$language" > "$stage_map.log" 2>&1 + ret=$? +fi +echo "end: $(date)" >> "$stage_map.time" + +if [ $ret -eq 0 ] +then + mv -f "$stage_map" "$output_map" + rm -f "$stage_map.log" +fi diff --git a/osm/main.sh b/osm/main.sh new file mode 100755 index 0000000..07e2725 --- /dev/null +++ b/osm/main.sh @@ -0,0 +1,50 @@ +#!/bin/bash +#set -x + +#configuration +config_file=/home/osm/convert/scripts/config.cfg + +# source a specified section from config +function source_section { + section_name="$1" + section_start="^\[$section_name\]$" + section_end="^\[/$section_name\]$" + + line_start=$(grep -n "$section_start" "$config_file" | cut -d: -f1) + line_end=$(expr $(grep -n "$section_end" "$config_file" | cut -d: -f1) - 1) + line_diff=$(expr $line_end - $line_start) + + tmpfile=$(mktemp) + head -n $line_end "$config_file" | tail -n $line_diff > "$tmpfile" + source "$tmpfile" + rm -f "$tmpfile" +} + +# general section from config +source_section "general" + +export osmosis_executable dir_maps dir_output dir_stage dir_poly filetype_osm filetype_map filetype_poly wget_limit_rate + +params_for_xarg_call="" + +#loop over the maps we want to generate +for act_country in $maps_to_generate +do + # reset defaults + wm_type="$default_type" + start_zoom="$default_start_zoom" + language="$default_language" + use_poly="$default_use_poly" + + source_section "$act_country" + + if [ "X$output_subdir" == "X" ] + then + output_subdir="." + fi + + params_for_xarg_call="$params_for_xarg_call \"$download_base_url/$download_map_path/$download_map_file.$filetype_osm\" \"$download_base_url/$download_poly_path/$download_poly_file.$filetype_poly\" \"$act_country.$filetype_map\" \"$wm_type\" \"$start_zoom\" \"$language\" \"$use_poly\" \"$output_subdir\"" +done + +#echo "$params_for_xarg_call" | xargs -n 8 -P $threads_to_start echo +echo "$params_for_xarg_call" | xargs -n 8 -P $threads_to_start $download_and_convert_script |