Skip to content

Commit 828cd22

Browse files
committed
Migrate message_data.tools.messagev
1 parent a9b2570 commit 828cd22

File tree

3 files changed

+249
-0
lines changed

3 files changed

+249
-0
lines changed

doc/api/tools-messagev.rst

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
MESSAGE V input files
2+
*********************
3+
4+
This document describes some of the file formats for the pre-:mod:`.ixmp` MESSAGE model, a.k.a. **MESSAGE V**, and code in :mod:`message_ix_models.tools.messagev` that reads these formats.
5+
6+
.. note:: See also the earlier :doc:`import_from_msgV_sqlite` for similar code/descriptions.
7+
8+
.. contents::
9+
:local:
10+
11+
``easemps3.free``: soft dynamic constraints
12+
-------------------------------------------
13+
14+
Each constraint is specified by a single row with the following, space-separated entries:
15+
16+
1. **Constraint type.** `mpa` (constraint on activity) or `mpc` (constraint on capacity).
17+
2. **Technology name.** Four-letter internal name/code of a technology, e.g. `uHap`.
18+
3. **Lower/upper bound.** Either `LO` (decline constraint) or `UP` (growth constraint).
19+
4. **Cost type.** One of:
20+
21+
- `lev`: levelized costs.
22+
- `abs`: absolute costs.
23+
- `var`: variable costs.
24+
25+
5. **Growth rate for step 1.** Percentage points of growth/decline at which the constraint becomes active.
26+
27+
6. **Additional cost for step 1.** Additional cost applied to activity/capacity growth or decline beyond the rate in #5. Depending on #4, specified:
28+
29+
- `lev`: in percentage points of the levelized cost of the technology.
30+
- `abs`, `var`: in absolute monetary value.
31+
32+
7. **Up to 4 additional pairs of 5 and 6.** Growth rates for successive constraints are cumulative.
33+
34+
An example:
35+
36+
.. code::
37+
38+
mpa uEAp UP lev 5 50 15 300000
39+
40+
Here the constraint relates to a growth constraint (UP) for activities (mpa) and the technology for which the constraint is to be extended is uEAp.
41+
The allowed rate of growth is increased by 5 %-points and each additional unit of output that can be produced costs 50 % of the levelized costs additional on top of the normal costs (i.e. the costs that result from building and using the additional capacity required for the additional production).
42+
43+
The second step increases the maximum growth rate further, by 15 %-points, but the costs are prohibitive (300000).
44+
45+
Soft constraints can be set for each technology individually. This can be done globally ("regions = all -glb") or for each region separately ("regions = cpa").
46+
47+
48+
API reference
49+
-------------
50+
51+
.. currentmodule:: message_ix_models.tools.messagev
52+
53+
.. automodule:: message_ix_models.tools.messagev
54+
:members:

doc/index.rst

+1
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ Commonly used classes may be imported directly from :mod:`message_ix_models`.
6969
api/report/index
7070
api/tools
7171
api/tools-costs
72+
api/tools-messagev
7273
api/data-sources
7374
api/util
7475
api/testing

message_ix_models/tools/messagev.py

+194
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
"""Tools for extracting data from MESSAGE V."""
2+
3+
import re
4+
from functools import lru_cache
5+
6+
import numpy as np
7+
import pandas as pd
8+
9+
10+
class CHNFile:
11+
"""Reader for MESSAGE V ``.chn`` files."""
12+
13+
index = {}
14+
15+
# FIXME reduce complexity from 15 to ≤14
16+
def __init__(self, path): # noqa: C901
17+
"""Parse .chn file."""
18+
19+
def _depth(str):
20+
return (len(str.replace("\t", " ")) - len(str.lstrip())) // 2
21+
22+
stack = []
23+
self.data = {}
24+
for line in open(path):
25+
if line.startswith("#"): # Comment
26+
pass
27+
elif len(stack) == 0: # New level
28+
name, level = line.split()
29+
stack.append((name, level.rstrip(":")))
30+
elif len(stack) == 1: # New commodity
31+
stack.append(tuple(line.split()))
32+
elif len(stack) == 2:
33+
if _depth(line) == 0 and line.strip() == "*": # End of level
34+
stack = []
35+
elif _depth(line) == 1: # New commodity
36+
stack[-1] = tuple(line.split())
37+
else:
38+
p_c = line.strip().rstrip(":")
39+
if p_c in ("Producers", "Consumers"): # Start of P/C block
40+
stack.append(p_c)
41+
pc_data = []
42+
elif p_c == "*": # Consecutive '*'
43+
pass
44+
elif len(stack) == 3:
45+
if _depth(line) == 2 and line.strip() == "*": # End of block
46+
# Store data
47+
if len(pc_data):
48+
key = tuple([stack[0][0], stack[1][0], stack[2]])
49+
self.data[key] = pc_data
50+
stack.pop(-1)
51+
else: # Data line
52+
# TODO parse: tec, level, code, commodity, {ts,c}, [data]
53+
pc_data.append(line.split())
54+
elif line == "*\n":
55+
stack.pop(-1)
56+
57+
58+
class DICFile:
59+
"""Reader for MESSAGE V ``.dic`` files."""
60+
61+
tec_code = {}
62+
code_tec = {}
63+
64+
def __init__(self, path=None):
65+
if path is None:
66+
return
67+
68+
for line in open(path):
69+
if line.startswith("#"):
70+
continue
71+
72+
tec, code = line.split()
73+
self.tec_code[tec] = code
74+
self.code_tec[code] = tec
75+
76+
def __getitem__(self, key):
77+
try:
78+
return self.code_tec[key]
79+
except KeyError:
80+
return self.tec_code[key]
81+
82+
83+
class INPFile:
84+
"""Reader for MESSAGE V ``.inp`` files."""
85+
86+
index = {}
87+
file = None
88+
years_re = re.compile(r"^timesteps:(( \d*)*)", re.MULTILINE)
89+
90+
def __init__(self, path):
91+
self.file = open(path)
92+
93+
# Index the file
94+
section = "_info"
95+
pos = self.file.tell()
96+
while True:
97+
line = self.file.readline()
98+
if line == "":
99+
break
100+
elif line == "*\n":
101+
self.index[section] = (pos, self.file.tell() - pos)
102+
section = None
103+
pos = self.file.tell()
104+
elif section is None:
105+
section = line.split()[0]
106+
107+
def get_section(self, name):
108+
start, len = self.index[name]
109+
self.file.seek(start)
110+
return self.file.read(len)
111+
112+
@lru_cache(1)
113+
def get_years(self):
114+
"""Return timesteps."""
115+
sec = self.get_section("_info")
116+
match = self.years_re.search(sec)
117+
return list(map(int, match.groups()[0].strip().split()))
118+
119+
params_with_source = "con1a con1c con2a inp minp moutp"
120+
ts_params = "ctime fom inv plf pll vom" + params_with_source
121+
scalar_params = {
122+
"annualize": int,
123+
"display": str,
124+
"fyear": int,
125+
"lyear": int,
126+
"hisc": float,
127+
"minp": float,
128+
}
129+
130+
def const_or_ts(self, line):
131+
param = line.pop(0)
132+
source = line.pop(0) if param in self.params_with_source else None
133+
134+
if param == "minp":
135+
line = ["c" if len(line) == 1 else "ts"] + line
136+
137+
kind = line.pop(0)
138+
if kind == "ts":
139+
elem = list(zip(self.get_years(), line))
140+
elif kind == "c":
141+
assert len(line) == 1
142+
143+
# # This line implements a fill-forward:
144+
# elem = [(year, line[0]) for year in self.get_years()]
145+
146+
# Single element
147+
elem = [(self.get_years()[0], line[0])]
148+
else:
149+
raise ValueError(param, source, kind, line)
150+
151+
# 'free' is a special value for bounds/constraints
152+
df = (
153+
pd.DataFrame(elem, columns=["year", "value"])
154+
.replace("free", np.nan)
155+
.astype({"value": float})
156+
)
157+
158+
# Add parameter name and source
159+
df["param"] = param
160+
df["source"] = source
161+
162+
return df
163+
164+
def parse_section(self, name):
165+
result = {}
166+
params = []
167+
168+
# Parse each line
169+
for line in map(str.split, self.get_section(name).split("\n")):
170+
if line in ([], ["*"]) or line[0].startswith("#"):
171+
# End of section, comment, or blank line
172+
continue
173+
174+
param = line[0]
175+
if param == name:
176+
# Start of the section
177+
result["extra"] = line[1:]
178+
elif param in "bda bdc":
179+
result["type"] = line.pop(1) # 'lo' or 'hi'
180+
params.append(self.const_or_ts(line))
181+
# elif param in 'mpa mpc':
182+
# # TODO implement this
183+
# continue
184+
elif param in self.ts_params:
185+
params.append(self.const_or_ts(line))
186+
elif param in self.scalar_params:
187+
assert len(line) == 2, line
188+
result[param] = self.scalar_params[param](line[1])
189+
190+
# Concatenate accumulated params to a single DataFrame
191+
if len(params):
192+
result["params"] = pd.concat(params, sort=False)
193+
194+
return result

0 commit comments

Comments
 (0)