Le mirroring des dépôts vers Github est maintenant natif dans Gitlab (et donc Framagit). Merci de passer à la fonction native pour que nous puissions couper notre solution maison. Détails sur https://docs.gitlab.com/ee/workflow/repository_mirroring.html#pushing-to-a-remote-repository (pensez à supprimer le webhook de notre solution maison).

Commit daa438c6 authored by piza's avatar piza

Merge branch 'master' of git.framasoft.org:pizaninja/OpenEarthView

Conflicts: osm2x3d/package.json
parents 26f59c8d d949c1a7
/nbproject/private/
/node_modules*
/trash/
/x2js-v1.1.5/
/server_old.js
/OsmGround.js
/osm2x3d.nfo
/myResources
# Intro
This module aims to convert Open Street Map XML data to geojson for 3D scene.
Available output data formats:
* GeoJSON
# Library
The project includes the following tool:
* OsmToGeoJSon: convert XML OSM data to extended GeoJSON data
```
__________________
| |
XML OSM --->| OsmSaxGeoJson | ---> GeoJSON
|__________________|
```
# Installation
```
$ npm install
```
# Web server
To start the web server:
```
$ web_server.sh
```
# Test
## Requests:
```
$ wget "localhost:8081/geojsontile?xtile=154394&ytile=197054&zoom=19" -O ESB19_geojson.json
```
## Automated tests
```
```
## Unit Tests
```
$ test/unit/OsmToGeoJsonMocha.js
```
# Licence
The MIT License (MIT)
Copyright (c) 2016 Clément Igonet
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the Software
is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
This diff is collapsed.
#!/bin/sh
node web_server.js --vv
{
"name": "osm2geojson",
"description": "Open Earth View project - Open Street Map to 3D formats converter",
"version": "0.0.1",
"devDependencies": {},
"dependencies": {
"body-parser": "~1.0.2",
"chai": "3.5.0",
"chai-fuzzy": "1.6.1",
"loglevel": "1.4.0",
"node-getopt": "0.2.3",
"node-xml-compare": "0.1.0",
"pkgfiles": "2.3.0",
"rgbcolor": "0.0.4",
"sax": "1.2.1",
"three": "0.76.1",
"turf": "2.0.2",
"xslt": "0.7.0"
},
"main": "./web_server.js",
"files": [
"lib",
"scripts",
"tools/junit.sh",
"package.json",
"README.md",
"web_server.js"
],
"author": "Clement IGONET <clement@igonet.fr>",
"keywords": [],
"repository": {
"type": "git",
"url": "git@git.framasoft.org:pizaninja/OpenEarthView.git"
},
"license": "MIT"
}
/**
Open Earth View - osm2x3d
The MIT License (MIT)
Copyright (c) 2016 Clément Igonet
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the Software
is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
var http = require('http');
// var httpSync = require('http-sync');
var url = require('url');
var querystring = require('querystring');
// var osmXmlToGeoJson = require('./lib/OsmXmlToGeoJson.js');
var osmXmlToGeoJson = require('./lib/OsmSaxGeoJson.js');
var log = require('loglevel');
opt = require('node-getopt').create([
// ['s' , '' , 'short option.'],
// ['' , 'long' , 'long option.'],
// ['S' , 'short-with-arg=ARG' , 'option with argument'],
// ['L' , 'long-with-arg=ARG' , 'long option with argument'],
// ['' , 'color[=COLOR]' , 'COLOR is optional'],
// ['m' , 'multi-with-arg=ARG+' , 'multiple option with argument'],
// ['' , 'no-comment'],
['', 'vvv', 'print in trace level'],
['', 'vv', 'print in debug level'],
['v', '', 'print in info level'],
// ['v' , 'version' , 'show version']
]) // create Getopt instance
.bindHelp() // bind option 'help' to default action
.parseSystem(); // parse command line
// log.setLevel("warn");
log.setLevel("warn");
if (opt.options.vvv) {
log.setLevel("trace");
} else if (opt.options.vv) {
log.setLevel("debug");
} else if (opt.options.v) {
log.setLevel("info");
}
// osmXmlToGeoJson.setLogLevel(log.getLevel());
var server = http.createServer(function(request, response) {
var page = url.parse(request.url).pathname;
log.trace("page:" + page);
if (page != '/geojsontile') {
log.warning('Will not serve path: ' + page);
response.writeHead(404, {
"Content-Type": "text/html"
});
response.end();
return;
}
var args = querystring.parse(url.parse(request.url).query);
log.trace(args);
if ('zoom' in args && 'xtile' in args && 'ytile' in args) {
log.info('Serving path: ' + page);
log.info('args: ' + JSON.stringify(args));
var factor = ('factor' in args) ? args.factor : 0;
// wget "http://www.openstreetmap.org/api/0.6/map?bbox=-73.9874267578125,40.74725696280421,-73.98605346679688,40.74829735476796" -O result.osm
// wget "http://www.openstreetmap.org/api/0.6/tiledata/18/77196/98527" -O tile.osm
// wget "http://a.tile.openstreetmap.org/18/77196/98527.png"
// wget "http://a.tile.openstreetmap.org/19/154393/197054.png"
// wget "localhost:8080/geojsontile?xtile=154394&ytile=197054&zoom=19" -O ESB19_geojson.json
if (args.zoom >= 16 && args.zoom <= 19) {
var loD = args.zoom - 15;
log.trace("Level of Details: " + loD);
// var options = {
// hostname: 'www.openstreetmap.org',
// port: 80,
// path: "/api/0.6/map?bbox=" +
// tile2long(+args.xtile, args.zoom) + "," + // left
// tile2lat(+args.ytile + 1, args.zoom) + "," + // bottom
// tile2long(+args.xtile + 1, args.zoom) + "," + // right
// tile2lat(+args.ytile, args.zoom), // top
// method: 'GET'
// };
var options = {
hostname: 'localhost',
port: 8082,
path: "/osmCache/osmXml?tile=" +
(args.zoom - factor) + "," +
Math.floor(args.xtile / (Math.pow(2, factor))) + "," +
Math.floor(args.ytile / (Math.pow(2, factor))),
method: 'GET'
};
log.info('for: ', args.zoom + ',' + args.xtile + ',' + args.ytile);
log.info('options.path:', options.path);
// var osmRequest = http.request(optionsOverpass, function(osmReadStream) {
log.info('Requesting: ' + options.hostname + ':' + options.port + options.path);
var osmRequest = http.request(options, function(osmReadStream) {
log.info('Get osm map response.')
var onX3dJsonConvert;
// var onGeoJsonConvert;
var myOptions = {
origin: [
tile2long(+args.xtile, args.zoom),
tile2lat(+args.ytile, args.zoom)
],
loD: loD,
geoJsonExtended: false,
zoom: args.zoom,
xtile: args.xtile,
ytile: args.ytile,
bounds: {
minlon: tile2long(+args.xtile, +args.zoom),
maxlon: tile2long(+args.xtile + 1, +args.zoom),
minlat: tile2lat(+args.ytile, +args.zoom),
maxlat: tile2lat(+args.ytile + 1, +args.zoom)
},
factor: factor,
logLevel: log.getLevel()
}
// log.info('myOptions:', JSON.stringify(myOptions));
// writeStreamCount++;
// if (writeStreamCount > 1) {
// conversionQueue.push(myOptions);
// } else {
converter(myOptions, osmReadStream, response);
// }
});
osmRequest.on('error', function(e) {
log.trace('problem with request: ' + e.message);
});
osmRequest.end();
}
} else {
log.warning("fallback");
response.writeHead(404, {
"Content-Type": "text/html"
});
response.end();
}
});
server.listen(8081);
var conversionQueue = [];
var converting = false;
var converter = function(options_, osmReadStream, response) {
if (converting) {
log.info('Conversion put in queue: ', options_.zoom, '/', options_.xtile, '/', options_.ytile);
conversionQueue.push({
options: options_,
osmReadStream: osmReadStream,
response: response
});
} else {
converting = true;
log.info('converting: ', converting);
var writeStream = osmXmlToGeoJson.convert(options_, function(geoJson) {
log.info('Ready to send: ', options_.zoom, '/', options_.xtile, '/', options_.ytile);
response.setHeader('Content-Type', 'application/json');
response.end(JSON.stringify(geoJson));
converting = false;
log.info('converting: ', converting);
while (!converting && conversionQueue.length > 0) {
var conversion = conversionQueue.pop();
converter(
conversion.options,
conversion.osmReadStream,
conversion.response);
}
});
if (writeStream !== undefined) {
log.info('Starting conversion: ', options_.zoom, '/', options_.xtile, '/', options_.ytile);
osmReadStream.pipe(writeStream, {
end: true
});
}
}
}
function tile2long(x, z) {
return (x / Math.pow(2, z) * 360 - 180);
}
function tile2lat(y, z) {
var n = Math.PI - 2 * Math.PI * y / Math.pow(2, z);
return (180 / Math.PI * Math.atan(0.5 * (Math.exp(n) - Math.exp(-n))));
}
......@@ -29,7 +29,7 @@ var sax = require("sax");
var turf = require('turf');
var log = require('loglevel');
log.setLevel("debug");
log.setLevel("warn");
function getRGB(osmColor) {
if (osmColor) {
......@@ -42,7 +42,7 @@ function getRGB(osmColor) {
return color.toRGB();
}
}
return "rgb(240,240,240)";
return undefined;
}
function heightToMeter(height) {
......@@ -169,14 +169,18 @@ function convert(options, onConvert) {
onRelation = false;
var way = {};
var relation = {};
bounds = groundBlock(
options.bounds,
((options && options.tile) ? options.tile : null));
blocks[blocks.length] = bounds;
xmlStream.on('opentag', function(node) {
var name = node.name,
attrs = node.attributes;
if (name === 'bounds') {
var tile = ((options && options.tile) ? options.tile : null);
bounds = groundBlock(attrs, tile);
blocks[blocks.length] = bounds;
} else if (name === 'node') {
// if (name === 'bounds') {
// var tile = ((options && options.tile) ? options.tile : null);
// bounds = groundBlock(attrs, tile);
// blocks[blocks.length] = bounds;
if (name === 'node') {
var id = +attrs.id;
var lat = attrs.lat;
var lon = attrs.lon;
......@@ -246,14 +250,17 @@ function convert(options, onConvert) {
relation.name = attrs.v;
break;
case 'building:part':
relation.isBldPart = (attrs.v === 'yes');
relation.isBld = (attrs.v === 'yes');
log.debug('relation ' + relation.id + ' is a buildingPart');
relation.isBldPart = true;
relation.isBld = true;
break;
case 'building':
relation.isBld = (attrs.v === 'yes');
relation.isBld = true;
log.debug('relation ' + relation.id + ' is a building');
log.debug('relation:' + JSON.stringify(relation));
break;
case 'type':
relation.isBld = (attrs.v === 'building');
relation.isBld = (attrs.v === 'building') ? true : relation.isBld;
break;
case 'height':
relation.osmBldPartHeight = heightToMeter(attrs.v);
......@@ -277,7 +284,7 @@ function convert(options, onConvert) {
// var roof = roofBlock(way);
// }
var geoBldPart = geoBldPartBlock(way);
if (way.isBld && !way.isBldPart) {
if (way.isBld) {
var geoBld = getGeoBuilding(geoBlds, +way.id);
geoBld.features[geoBld.features.length] = geoBldPart;
// if (roof && options.loD > 1 && options.geoJsonExtended && options.geoJsonExtended == true) {
......@@ -300,9 +307,13 @@ function convert(options, onConvert) {
// },
// if (geoBld.features[0].properties.id == "249680748") {
if (turf.inside(turf.centroid(geoBld), bounds)) {
log.debug('building ' + way.id + ' is inside bounds.');
// if (way.isBldPart || turf.inside(turf.centroid(geoBld), bounds)) {
blocks[blocks.length] = JSON.parse(JSON.stringify(geoBld));
// }
} else {
log.debug('building ' + way.id + ' is not inside bounds.');
}
//
// }
......@@ -315,38 +326,44 @@ function convert(options, onConvert) {
// console.log('geoBld:', JSON.stringify(geoBld));
// }
} else {
// if (turf.inside(turf.centroid(geoBldPart), bounds)) {
geoBldParts[+way.id] = geoBldPart;
// }
if (turf.inside(turf.centroid(geoBldPart), bounds)) {
log.debug('way ' + way.id + ' is inside bounds.');
geoBldParts[+way.id] = geoBldPart;
} else {
log.debug('way ' + way.id + ' is not inside bounds.');
}
// if (roof) {
// roofs[+way.id] = roof;
// }
}
} else if (name === 'relation') {
log.debug('relation:' + JSON.stringify(relation));
// <tag k="type" v="building"/>
if (!relation.isBld) {
removeBuilding(geoBlds, +relation.id);
log.debug("relation is not a building.");
log.debug('relation ' + relation.id + ' is not a building.');
} else {
log.debug("relation is a building.");
log.debug('relation ' + relation.id + ' is a building.');
var geoBld = getGeoBuilding(geoBlds, relation.id);
if (relation.osmBldPartHeight !== undefined) {
geoBld.properties.name = relation.name;
for (var i = 0; i < geoBld.features.length; i++) {
var geoBldPart = geoBld.features[i];
if (geoBldPart.properties.height !== undefined) {
geoBldPart.properties.height = relation.osmBldPartHeight;
}
if (geoBldPart.properties.minHeight === undefined) {
geoBldPart.properties.minHeight = relation.minHeight;
}
// if (relation.osmBldPartHeight !== undefined) {
relation.osmBldPartHeight = (relation.osmBldPartHeight !== undefined) ? relation.osmBldPartHeight : 10;
geoBld.properties.name = relation.name;
for (var i = 0; i < geoBld.features.length; i++) {
var geoBldPart = geoBld.features[i];
// geoBldPart.properties.height = (relation.osmBldPartHeight !== undefined) ? relation.osmBldPartHeight : 10;
if (geoBldPart.properties.height === undefined) {
geoBldPart.properties.height = relation.osmBldPartHeight;
}
// geoBldPart.properties.minHeight = (relation.minHeight !== undefined) ? relation.minHeight : 0;
if (geoBldPart.properties.minHeight === undefined) {
geoBldPart.properties.minHeight = relation.minHeight;
}
}
if (turf.inside(turf.centroid(geoBld), bounds)) {
blocks[blocks.length] = JSON.parse(JSON.stringify(geoBld));
}
// }
// if (turf.inside(turf.centroid(geoBld), bounds)) {
blocks[blocks.length] = JSON.parse(JSON.stringify(geoBld));
// }
log.debug('Add block:' + JSON.stringify(geoBld));
}
onRelation = false;
} else if (name === 'osm') {
......
......@@ -12,10 +12,10 @@
"node-xml-compare": "",
"pkgfiles": "",
"rgbcolor": "0.0.4",
"sax": "",
"three": "",
"turf": "",
"xslt": ""
"sax": "1.2.1",
"three": "0.76.1",
"turf": "2.0.2",
"xslt": "0.7.0"
},
"main": "./web_server.js",
"scripts": {
......
......@@ -46,6 +46,7 @@ opt = require('node-getopt').create([
.bindHelp() // bind option 'help' to default action
.parseSystem(); // parse command line
// log.setLevel("warn");
log.setLevel("warn");
if (opt.options.debug) {
log.setLevel("debug");
......@@ -97,6 +98,7 @@ var server = http.createServer(function(request, response) {
var args = querystring.parse(url.parse(request.url).query);
log.debug(args);
if ('format' in args && 'zoom' in args && 'xtile' in args && 'ytile' in args) {
var factor = ('factor' in args) ? args.factor : 0;
// wget "http://www.openstreetmap.org/api/0.6/map?bbox=-73.9874267578125,40.74725696280421,-73.98605346679688,40.74829735476796" -O result.osm
// wget "http://www.openstreetmap.org/api/0.6/tiledata/18/77196/98527" -O tile.osm
// wget "http://www.openearthview.net/osm2x3d.php?zoom=18&xtile=77196&ytile=98527" -O ESB18_old.x3d
......@@ -124,12 +126,17 @@ var server = http.createServer(function(request, response) {
// tile2lat(+args.ytile, args.zoom), // top
// method: 'GET'
// };
var options = {
hostname: 'localhost',
port: 8082,
path: "/osmCache/osmXml?tile=" + args.zoom + "," + args.xtile + "," + args.ytile,
path: "/osmCache/osmXml?tile=" +
(args.zoom - factor) + "," +
Math.floor(args.xtile / (Math.pow(2, factor))) + "," +
Math.floor(args.ytile / (Math.pow(2, factor))),
method: 'GET'
};
console.log('options:', options);
// var osmRequest = http.request(optionsOverpass, function(osmReadStream) {
var osmRequest = http.request(options, function(osmReadStream) {
log.debug('Get osm map response.')
......@@ -145,8 +152,15 @@ var server = http.createServer(function(request, response) {
geoJsonExtended: false,
zoom: args.zoom,
xtile: args.xtile,
ytile: args.ytile
ytile: args.ytile,
bounds: {
minlon: tile2long(+args.xtile, +args.zoom),
maxlon: tile2long(+args.xtile + 1, +args.zoom),
minlat: tile2lat(+args.ytile, +args.zoom),
maxlat: tile2lat(+args.ytile + 1, +args.zoom)
}
}
// console.log('myOptions:', JSON.stringify(myOptions));
var myWriteStream;
switch (args.format) {
case "osm":
......
#!/bin/sh
curDir=$(dirname "$0")
mkdir -p ~/.osmcache
flock -xn ~/.osmcache/flock.pid node ${curDir}/server.js --cacheDir=~/.osmcache -d
var path = require('path');
var fs = require('fs');
var os = require('os');
fs.mkdirParentSync = function(dirPath, mode) {
console.log("dirPath: " + dirPath);
//Call the standard fs.mkdir
try {
fs.mkdirSync(dirPath, mode);
} catch (e) {
console.log("error: " + e.code);
if (e.code === 'EEXIST') {
// path already exists
} else if (e.code === 'ENOENT') {
fs.mkdirParentSync(path.dirname(dirPath), mode);
fs.mkdirParentSync(dirPath, mode);
} else {
throw e;
}
}
};
var http = require('http');
var log = require('loglevel');
var querystring = require('querystring');
var url = require('url');
var CACHE_DIR = os.homedir() + '/.overpass/cache'
var opt = require('node-getopt').create([
['c', 'cacheDir=ARG', 'Default cache dir is ' + CACHE_DIR],
['d', 'debug', 'print in debug level'],
]).bindHelp().parseSystem();
if (cacheDir === undefined) {
var cacheDir = CACHE_DIR;
}
fs.mkdirParentSync(cacheDir);
log.setLevel("warn");
if (opt.options.debug) {
log.setLevel("debug");
}
var server = http.createServer(function(request, response) {
var page = url.parse(request.url).pathname;
log.debug("page:" + page);
// www.openstreetmap.org/api/0.6/map?bbox=left,bottom,right,top
// localhost:8082/osmCache/osmXml?tile=zoom,xtile,ytile
// if (page != '/osmCache/osmXml' && page != '/osmCache/osmJson' ) {
if (page != '/osmCache/osmXml') {
response.writeHead(404, {
"Content-Type": "text/html"
});
response.end();
return;
}
var args = querystring.parse(url.parse(request.url).query);
log.debug(args);
if ('tile' in args) {
var tile = args.tile.split(',');
var zoom = tile[0];
var xtile = tile[1];
var ytile = tile[2];
var cacheFile = cacheDir + "/" + zoom + "/" + xtile + "/" + ytile + ".xml";
fs.stat(
cacheFile,
function(err, stats) {
if (err !== null && err.code === 'ENOENT') {
// || !stats.isFile()
// TODO: reset file
log.debug('Can not access file ' + cacheFile);
log.debug('err: ' + err);
// Get data from external overpass API
// and write it to cache file
var myPath = "/api/0.6/map?bbox=" +
tile2long(+xtile, zoom) + "," +
tile2lat(+ytile + 1, zoom) + "," +
tile2long(+xtile + 1, zoom) + "," +
tile2lat(+ytile, zoom);
console.log("myPath: " + myPath);
var osmRequest = http.request({
hostname: 'www.openstreetmap.org',
port: 80,
path: myPath,
method: 'GET'
}, function(osmReadStream) {
console.log("pipe to :" + cacheFile)
fs.mkdirParentSync(path.dirname(cacheFile));
osmReadStream.pipe(fs.createWriteStream(cacheFile), {
end: true
});
osmReadStream.on('end', () => {
response.writeHead(200, {
'Content-Type': 'text/xml'
});
fs.createReadStream(cacheFile).pipe(response, {
end: true
});
});
});
osmRequest.on('error', function(e) {
log.debug('problem with request: ' + e.message);
});
osmRequest.end();
} else {
response.writeHead(200, {
'Content-Type': 'text/xml',
'Content-Length': stats.size
});
fs.createReadStream(cacheFile).pipe(response, {
end: true
});
}
// Return osm data
}
);
} else {
// fallback;
response.writeHead(404, {
"Content-Type": "text/html"
});
response.end();
}
});
server.listen(8082);
function tile2long(x, z) {
return (x / Math.pow(2, z) * 360) % 360 - 180;
}
function tile2lat(y, z) {
var n = Math.PI - 2 * Math.PI * y / Math.pow(2, z);
return (180 / Math.PI * Math.atan(0.5 * (Math.exp(n) - Math.exp(-n))));
}
# Intro
This module aims to put Open Earth View geojson data in local cache.
# Installation
```
$ npm install
```
# Web server
To start the web server: