Merge branch 'master' of github.com:lookshe/scripts
This commit is contained in:
lookshe
commit
61672bf575
7 changed files with 1505 additions and 0 deletions
35
download_webscraper.pl
Normal file
35
download_webscraper.pl
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
#!/usr/bin/perl
|
||||
|
||||
#use strict;
|
||||
#use warnings;
|
||||
use Web::Scraper;
|
||||
use URI;
|
||||
use HTML::Entities;
|
||||
use Encode;
|
||||
use URI::Escape;
|
||||
use LWP::UserAgent;
|
||||
|
||||
my $scrap;
|
||||
|
||||
my $wikiurl = "http://www.bildung-lsa.de/unterricht/zentrale_leistungserhebungen__schriftliche_pruefungen__zentrale_klassenarbeiten__vergleichsarbeiten____/schriftliche_abiturpruefung.html";
|
||||
|
||||
my $ua = new LWP::UserAgent;
|
||||
my $req = HTTP::Request->new('GET', $wikiurl);
|
||||
my $res = $ua->request($req);
|
||||
my $url = $res->request->uri;
|
||||
|
||||
binmode(STDOUT, ":utf8");
|
||||
|
||||
|
||||
$scrap = scraper {
|
||||
process '//a[@class="subjectlink"]', 'href[]' => '@href';
|
||||
};
|
||||
$url = URI->new($wikiurl);
|
||||
|
||||
my $res = $scrap->scrape($url);
|
||||
my $href = $res->{'href'};
|
||||
for ($i = 0; $i <= $#$href; $i++)
|
||||
{
|
||||
my $url = $$href[$i];
|
||||
system("wget -q \"$url\"");
|
||||
}
|
||||
6
externalIp.pl
Normal file
6
externalIp.pl
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
#!/usr/bin/perl
|
||||
|
||||
($ip=`wget -q -O - http://checkip.dyndns.org`)=~s/\n$//;
|
||||
$ip=~s/.*[^\d](\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[^\d].*/$1/;
|
||||
($date=`date`)=~s/\n$//;
|
||||
print "$date\n$ip\n";
|
||||
21
ipchanges.pl
Normal file
21
ipchanges.pl
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
#!/usr/bin/perl
|
||||
|
||||
my $file = "/home/lookshe/myips.txt";
|
||||
|
||||
open(FILE, $file) || die("unable to open $file!");
|
||||
my @lines = <FILE>;
|
||||
close(FILE);
|
||||
|
||||
for ($i = 1; $i <= $#lines-2; $i+=2)
|
||||
{
|
||||
my $ip = $lines[$i];
|
||||
chomp($ip);
|
||||
my $nextip = $lines[$i+2];
|
||||
chomp($nextip);
|
||||
if ($ip !~ m/^$nextip$/)
|
||||
{
|
||||
my $time = $lines[$i+1];
|
||||
chomp($time);
|
||||
print "$time: $ip -> $nextip\n";
|
||||
}
|
||||
}
|
||||
1272
osm/config.cfg
Normal file
1272
osm/config.cfg
Normal file
File diff suppressed because it is too large
Load diff
53
osm/config.cfg_bak
Normal file
53
osm/config.cfg_bak
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
[general]
|
||||
osmosis_executable=/home/osm/convert/tools/osmosis-0.40.1/bin/osmosis
|
||||
download_and_convert_script=/home/osm/convert/scripts/download_and_convert.sh
|
||||
|
||||
dir_maps=/home/osm/convert/maps
|
||||
dir_output=/home/osm/convert/output
|
||||
dir_stage=/home/osm/convert/stage
|
||||
dir_poly=/home/osm/convert/poly
|
||||
|
||||
default_type=hd
|
||||
default_start_zoom=14
|
||||
default_language=en
|
||||
default_use_poly=true
|
||||
filetype_osm=osm.pbf
|
||||
filetype_map=map
|
||||
filetype_poly=poly
|
||||
|
||||
threads_to_start=2
|
||||
|
||||
download_base_url=http://download.geofabrik.de
|
||||
wget_limit_rate=20m
|
||||
|
||||
maps_to_generate="germany berlin hamburg"
|
||||
[/general]
|
||||
|
||||
[possible_values]
|
||||
download_map_path=relative path to map-file to downloaded
|
||||
download_map_file=filename of the map-file to download
|
||||
download_poly_path=relativ path to poly-file to download
|
||||
download_poly_file=filename of poly-file to download
|
||||
use_poly=override $default_use_poly
|
||||
type=overwrite $default_type (hd|ram)
|
||||
start_zoom=overwrite $default_start_zoom
|
||||
language=override $default_language
|
||||
[/possible_values]
|
||||
|
||||
[germany]
|
||||
download_map_path=europe
|
||||
download_map_file=germany-latest
|
||||
download_poly_path=europe
|
||||
download_poly_file=germany
|
||||
use_poly=false
|
||||
[/germany]
|
||||
|
||||
[berlin]
|
||||
download_poly_path=europe/germany
|
||||
download_poly_file=berlin
|
||||
[/berlin]
|
||||
|
||||
[hamburg]
|
||||
download_poly_path=europe/germany
|
||||
download_poly_file=hamburg
|
||||
[/hamburg]
|
||||
68
osm/download_and_convert.sh
Executable file
68
osm/download_and_convert.sh
Executable file
|
|
@ -0,0 +1,68 @@
|
|||
#!/bin/bash
|
||||
#set -x
|
||||
|
||||
if [ $# -ne 8 ]
|
||||
then
|
||||
echo "wrong argument count"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
url_map="$1"
|
||||
url_map_filename="$(basename $url_map)"
|
||||
download_map="$dir_maps/$url_map_filename"
|
||||
url_poly="$2"
|
||||
url_poly_filename="$(basename $url_poly)"
|
||||
download_poly="$dir_poly/$url_poly_filename"
|
||||
mapfile="$3"
|
||||
wm_type="$4"
|
||||
start_zoom="$5"
|
||||
language="$6"
|
||||
use_poly="$7"
|
||||
output_subdir="$8"
|
||||
stage_map="$dir_stage/$mapfile"
|
||||
mkdir -p "$dir_output/$output_subdir/"
|
||||
output_map="$dir_output/$output_subdir/$mapfile"
|
||||
|
||||
|
||||
# cause of some timing problems we wait a random number of 1-10 seconds
|
||||
sleep $[($RANDOM % 10) + 1]
|
||||
|
||||
# check if we need to download
|
||||
if [ ! -e "$download_map" ]
|
||||
then
|
||||
# wait if another process is downloading it
|
||||
if [ -e "$download_map.tmp" ]
|
||||
then
|
||||
while [ ! -e "$download_map" ]
|
||||
do
|
||||
sleep 10
|
||||
done
|
||||
# download if we are the only one
|
||||
else
|
||||
touch "$download_map.tmp"
|
||||
wget -q --limit-rate=$wget_limit_rate "$url_map" -O "$download_map.tmp"
|
||||
mv -f "$download_map.tmp" "$download_map"
|
||||
fi
|
||||
fi
|
||||
|
||||
# download poly-file
|
||||
wget -q --limit-rate=$wget_limit_rate "$url_poly" -O "$download_poly"
|
||||
|
||||
echo "start: $(date)" >> "$stage_map.time"
|
||||
if [ "$use_poly" = "true" ]
|
||||
then
|
||||
echo "command: $osmosis_executable --rb \"$download_map\" --bp clipIncompleteEntities=true file=\"$download_poly\" --mw file=\"$stage_map\" type=\"$wm_type\" map-start-zoom=\"$start_zoom\" preferred-language=\"$language\"" >> "$stage_map.time"
|
||||
$osmosis_executable --rb "$download_map" --bp clipIncompleteEntities=true file="$download_poly" --mw file="$stage_map" type="$wm_type" map-start-zoom="$start_zoom" preferred-language="$language" > "$stage_map.log" 2>&1
|
||||
ret=$?
|
||||
else
|
||||
echo "command: $osmosis_executable --rb \"$download_map\" --mw file=\"$stage_map\" type=\"$wm_type\" map-start-zoom=\"$start_zoom\" preferred-language=\"$language\"" >> "$stage_map.time"
|
||||
$osmosis_executable --rb "$download_map" --mw file="$stage_map" type="$wm_type" map-start-zoom="$start_zoom" preferred-language="$language" > "$stage_map.log" 2>&1
|
||||
ret=$?
|
||||
fi
|
||||
echo "end: $(date)" >> "$stage_map.time"
|
||||
|
||||
if [ $ret -eq 0 ]
|
||||
then
|
||||
mv -f "$stage_map" "$output_map"
|
||||
rm -f "$stage_map.log"
|
||||
fi
|
||||
50
osm/main.sh
Executable file
50
osm/main.sh
Executable file
|
|
@ -0,0 +1,50 @@
|
|||
#!/bin/bash
|
||||
#set -x
|
||||
|
||||
#configuration
|
||||
config_file=/home/osm/convert/scripts/config.cfg
|
||||
|
||||
# source a specified section from config
|
||||
function source_section {
|
||||
section_name="$1"
|
||||
section_start="^\[$section_name\]$"
|
||||
section_end="^\[/$section_name\]$"
|
||||
|
||||
line_start=$(grep -n "$section_start" "$config_file" | cut -d: -f1)
|
||||
line_end=$(expr $(grep -n "$section_end" "$config_file" | cut -d: -f1) - 1)
|
||||
line_diff=$(expr $line_end - $line_start)
|
||||
|
||||
tmpfile=$(mktemp)
|
||||
head -n $line_end "$config_file" | tail -n $line_diff > "$tmpfile"
|
||||
source "$tmpfile"
|
||||
rm -f "$tmpfile"
|
||||
}
|
||||
|
||||
# general section from config
|
||||
source_section "general"
|
||||
|
||||
export osmosis_executable dir_maps dir_output dir_stage dir_poly filetype_osm filetype_map filetype_poly wget_limit_rate
|
||||
|
||||
params_for_xarg_call=""
|
||||
|
||||
#loop over the maps we want to generate
|
||||
for act_country in $maps_to_generate
|
||||
do
|
||||
# reset defaults
|
||||
wm_type="$default_type"
|
||||
start_zoom="$default_start_zoom"
|
||||
language="$default_language"
|
||||
use_poly="$default_use_poly"
|
||||
|
||||
source_section "$act_country"
|
||||
|
||||
if [ "X$output_subdir" == "X" ]
|
||||
then
|
||||
output_subdir="."
|
||||
fi
|
||||
|
||||
params_for_xarg_call="$params_for_xarg_call \"$download_base_url/$download_map_path/$download_map_file.$filetype_osm\" \"$download_base_url/$download_poly_path/$download_poly_file.$filetype_poly\" \"$act_country.$filetype_map\" \"$wm_type\" \"$start_zoom\" \"$language\" \"$use_poly\" \"$output_subdir\""
|
||||
done
|
||||
|
||||
#echo "$params_for_xarg_call" | xargs -n 8 -P $threads_to_start echo
|
||||
echo "$params_for_xarg_call" | xargs -n 8 -P $threads_to_start $download_and_convert_script
|
||||
Loading…
Reference in a new issue